Merge branch 'candidate-8.4.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

esp/src/package-lock.json

@@ -806,12 +806,12 @@
       }
     },
     "@hpcc-js/comms": {
-      "version": "2.62.0",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.62.0.tgz",
-      "integrity": "sha512-J7usN3Ao3NC7WA1LfREf6VsKW6AKJpb0PyjDa2WJJJClEnStwatP4W8p0q1CmXf1Vj5tfW28/V+lsJqAUYbk4w==",
+      "version": "2.64.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.64.0.tgz",
+      "integrity": "sha512-XWO7cGR393vA3a14kjKMBZdxptlfzLAS0d8nixL152uQlASjXErJ9E3vL+fcZ1sy+yZ5Gy17du3VOL3nGtwDYw==",
       "requires": {
         "@hpcc-js/ddl-shim": "^2.17.19",
-        "@hpcc-js/util": "^2.39.0",
+        "@hpcc-js/util": "^2.40.0",
         "@xmldom/xmldom": "0.7.2",
         "abort-controller": "3.0.0",
         "node-fetch": "2.6.5",

esp/src/package.json

@@ -40,7 +40,7 @@
     "@hpcc-js/chart": "^2.67.0",
     "@hpcc-js/codemirror": "^2.49.0",
     "@hpcc-js/common": "^2.57.0",
-    "@hpcc-js/comms": "^2.62.0",
+    "@hpcc-js/comms": "2.64.0",
     "@hpcc-js/dataflow": "^3.0.1",
     "@hpcc-js/eclwatch": "^2.62.0",
     "@hpcc-js/graph": "^2.69.0",

esp/src/src-react/components/FileBlooms.tsx

@@ -1,10 +1,7 @@
 import * as React from "react";
 import { CommandBar, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import * as Observable from "dojo/store/Observable";
-import { Memory } from "src/Memory";
 import nlsHPCC from "src/nlsHPCC";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { useFile } from "../hooks/file";
 import { HolyGrail } from "../layouts/HolyGrail";
 
@@ -19,32 +16,32 @@ export const FileBlooms: React.FunctionComponent<FileBloomsProps> = ({
 }) => {
 
     const [file, , , refreshData] = useFile(cluster, logicalFile);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("FieldNames")));
-    const [Grid, , refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, , copyButtons] = useFluentGrid({
+        data,
+        primaryID: "FieldNames",
         sort: [{ attribute: "FieldNames", "descending": false }],
         filename: "fileBlooms",
         columns: {
-            FieldNames: { label: nlsHPCC.FieldNames, sortable: true, },
-            Limit: { label: nlsHPCC.Limit, sortable: true, },
-            Probability: { label: nlsHPCC.Probability, sortable: true, }
+            FieldNames: { label: nlsHPCC.FieldNames, sortable: true, width: 320 },
+            Limit: { label: nlsHPCC.Limit, sortable: true, width: 180 },
+            Probability: { label: nlsHPCC.Probability, sortable: true, width: 180 },
         }
     });
 
     React.useEffect(() => {
         const fileBlooms = file?.Blooms?.DFUFileBloom;
         if (fileBlooms) {
-            store.setData(fileBlooms.map(bloom => {
+            setData(fileBlooms.map(bloom => {
                 return {
                     ...bloom,
                     FieldNames: bloom?.FieldNames?.Item[0] || "",
                 };
             }));
-            refreshTable();
         }
-    }, [file?.Blooms?.DFUFileBloom, refreshTable, store]);
+    }, [file?.Blooms?.DFUFileBloom]);
 
     //  Command Bar  ---
     const buttons = React.useMemo((): ICommandBarItemProps[] => [

esp/src/src-react/components/FileDetails.tsx

@@ -54,19 +54,19 @@ export const FileDetails: React.FunctionComponent<FileDetailsProps> = ({
                     : <></>
                 }
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.Contents} itemKey="Contents" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.Contents} itemKey="contents" style={pivotItemStyle(size, 0)}>
                 <Result cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.DataPatterns} itemKey="DataPatterns" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.DataPatterns} itemKey="datapatterns" style={pivotItemStyle(size, 0)}>
                 <DataPatterns cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.ECL} itemKey="ECL" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.ECL} itemKey="ecl" style={pivotItemStyle(size, 0)}>
                 <ECLSourceEditor text={file?.Ecl} readonly={true} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.DEF} itemKey="DEF" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.DEF} itemKey="def" style={pivotItemStyle(size, 0)}>
                 <XMLSourceEditor text={defFile} readonly={true} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.XML} itemKey="XML" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.XML} itemKey="xml" style={pivotItemStyle(size, 0)}>
                 <XMLSourceEditor text={xmlFile} readonly={true} />
             </PivotItem>
             {file?.isSuperfile
@@ -77,22 +77,22 @@ export const FileDetails: React.FunctionComponent<FileDetailsProps> = ({
                     <SuperFiles cluster={cluster} logicalFile={logicalFile} />
                 </PivotItem>
             }
-            <PivotItem headerText={nlsHPCC.FileParts} itemKey="FileParts" itemCount={file?.fileParts().length ?? 0} style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.FileParts} itemKey="parts" itemCount={file?.fileParts().length ?? 0} style={pivotItemStyle(size, 0)}>
                 <FileParts cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
             <PivotItem headerText={nlsHPCC.Queries} itemKey="queries" style={pivotItemStyle(size, 0)}>
                 <Queries filter={{ FileName: logicalFile }} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.Graphs} itemKey="Graphs" itemCount={file?.Graphs?.ECLGraph?.length} headerButtonProps={{ disabled: isDFUWorkunit }} style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.Graphs} itemKey="graphs" itemCount={file?.Graphs?.ECLGraph?.length} headerButtonProps={{ disabled: isDFUWorkunit }} style={pivotItemStyle(size, 0)}>
                 <FileDetailsGraph cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.History} itemKey="History" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.History} itemKey="history" style={pivotItemStyle(size, 0)}>
                 <FileHistory cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.Blooms} itemKey="Blooms" itemCount={file?.Blooms?.DFUFileBloom?.length} style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.Blooms} itemKey="blooms" itemCount={file?.Blooms?.DFUFileBloom?.length} style={pivotItemStyle(size, 0)}>
                 <FileBlooms cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
-            <PivotItem headerText={nlsHPCC.ProtectBy} itemKey="ProtectBy" style={pivotItemStyle(size, 0)}>
+            <PivotItem headerText={nlsHPCC.ProtectBy} itemKey="protectby" style={pivotItemStyle(size, 0)}>
                 <ProtectedBy cluster={cluster} logicalFile={logicalFile} />
             </PivotItem>
         </Pivot>

esp/src/src-react/components/FileDetailsGraph.tsx

@@ -1,10 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { Memory, Observable } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Image, Link } from "@fluentui/react";
 import * as Utility from "src/Utility";
 import nlsHPCC from "src/nlsHPCC";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { useFile } from "../hooks/file";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
@@ -37,11 +35,12 @@ export const FileDetailsGraph: React.FunctionComponent<FileDetailsGraphProps> =
 
     const [file, , , refreshData] = useFile(cluster, logicalFile);
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("Name")));
-    const [Grid, selection, refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "Name",
         filename: "graphs",
         columns: {
             col1: selector({
@@ -51,7 +50,11 @@ export const FileDetailsGraph: React.FunctionComponent<FileDetailsGraphProps> =
             Name: {
                 label: nlsHPCC.Name, sortable: true,
                 formatter: function (Name, row) {
-                    return Utility.getImageHTML(getStateImageName(row)) + `&nbsp;<a href='#/workunits/${file?.Wuid}/graphs/${Name}' onClick='return false;' class='dgrid-row-url'>${Name}</a>`;
+                    return <>
+                        <Image src={Utility.getImageURL(getStateImageName(row))} />
+                        &nbsp;
+                        <Link href={`#/workunits/${row?.Wuid}/metrics/${Name}`}>{Name}</Link>
+                    </>;
                 }
             }
         }
@@ -68,10 +71,10 @@ export const FileDetailsGraph: React.FunctionComponent<FileDetailsGraphProps> =
             key: "open", text: nlsHPCC.Open, disabled: !uiState.hasSelection, iconProps: { iconName: "WindowEdit" },
             onClick: () => {
                 if (selection.length === 1) {
-                    window.location.href = `#/workunits/${file?.Wuid}/graphs/${selection[0].Name}`;
+                    window.location.href = `#/workunits/${file?.Wuid}/metrics/${selection[0].Name}`;
                 } else {
                     for (let i = 0; i < selection.length; ++i) {
-                        window.open(`#/workunits/${file?.Wuid}/graphs/${selection[i].Name}`, "_blank");
+                        window.open(`#/workunits/${file?.Wuid}/metrics/${selection[i].Name}`, "_blank");
                     }
                 }
             }
@@ -86,17 +89,17 @@ export const FileDetailsGraph: React.FunctionComponent<FileDetailsGraphProps> =
     }, [selection]);
 
     React.useEffect(() => {
-        store.setData((file?.Graphs?.ECLGraph || []).map(item => {
+        setData((file?.Graphs?.ECLGraph || []).map(item => {
             return {
                 Name: item,
                 Label: "",
                 Completed: "",
                 Time: 0,
-                Type: ""
+                Type: "",
+                Wuid: file?.Wuid
             };
         }));
-        refreshTable();
-    }, [store, file?.Graphs?.ECLGraph, refreshTable]);
+    }, [file?.Graphs?.ECLGraph, file?.Wuid]);
 
     return <HolyGrail
         header={<CommandBar items={buttons} farItems={copyButtons} />}

esp/src/src-react/components/FileHistory.tsx

@@ -1,11 +1,9 @@
 import * as React from "react";
 import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { Memory, Observable } from "src/Memory";
 import nlsHPCC from "src/nlsHPCC";
 import { useConfirm } from "../hooks/confirm";
 import { useFileHistory } from "../hooks/file";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 
@@ -21,10 +19,11 @@ export const FileHistory: React.FunctionComponent<FileHistoryProps> = ({
 
     //  Grid ---
     const [history, eraseHistory, refreshData] = useFileHistory(cluster, logicalFile);
+    const [data, setData] = React.useState<any[]>([]);
 
-    const store = useConst(new Observable(new Memory("Name")));
-    const [Grid, _selection, refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, _selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "__hpcc_id",
         sort: [{ attribute: "Name", "descending": false }],
         filename: "filehistory",
         columns: {
@@ -45,9 +44,8 @@ export const FileHistory: React.FunctionComponent<FileHistoryProps> = ({
     });
 
     React.useEffect(() => {
-        store.setData(history);
-        refreshTable();
-    }, [history, refreshTable, store]);
+        setData(history.map((item, idx) => ({ ...item, __hpcc_id: idx })));
+    }, [history]);
 
     //  Command Bar  ---
     const buttons: ICommandBarItemProps[] = React.useMemo(() => [

esp/src/src-react/components/FileParts.tsx

@@ -1,11 +1,8 @@
 import * as React from "react";
 import { ICommandBarItemProps, CommandBar } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
 import { format as d3Format } from "@hpcc-js/common";
-import * as Observable from "dojo/store/Observable";
-import { Memory } from "src/Memory";
 import nlsHPCC from "src/nlsHPCC";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { useFile } from "../hooks/file";
 import { HolyGrail } from "../layouts/HolyGrail";
 
@@ -22,26 +19,27 @@ export const FileParts: React.FunctionComponent<FilePartsProps> = ({
 }) => {
 
     const [file, , , refreshData] = useFile(cluster, logicalFile);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("Id")));
-    const [Grid, _selection, refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, _selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "Id",
         sort: [{ attribute: "Id", "descending": false }],
         filename: "fileParts",
         columns: {
-            Id: { label: nlsHPCC.Part, sortable: true, },
-            Copy: { label: nlsHPCC.Copy, sortable: true, },
-            Ip: { label: nlsHPCC.IP, sortable: true, },
-            Cluster: { label: nlsHPCC.Cluster, sortable: true, },
-            PartsizeInt64: { label: nlsHPCC.Size, sortable: true, },
-            CompressedSize: { label: nlsHPCC.CompressedSize, sortable: true, },
+            Id: { label: nlsHPCC.Part, sortable: true, width: 80 },
+            Copy: { label: nlsHPCC.Copy, sortable: true, width: 80 },
+            Ip: { label: nlsHPCC.IP, sortable: true, width: 80 },
+            Cluster: { label: nlsHPCC.Cluster, sortable: true, width: 480 },
+            PartsizeInt64: { label: nlsHPCC.Size, sortable: true, width: 120 },
+            CompressedSize: { label: nlsHPCC.CompressedSize, sortable: true, width: 120 },
         }
     });
 
     React.useEffect(() => {
         const fileParts = file?.fileParts() ?? [];
-        store.setData(fileParts.map(part => {
+        setData(fileParts.map(part => {
             return {
                 Id: part.Id,
                 Copy: part.Copy,
@@ -51,8 +49,7 @@ export const FileParts: React.FunctionComponent<FilePartsProps> = ({
                 CompressedSize: part.CompressedSize ? formatNum(part.CompressedSize) : ""
             };
         }));
-        refreshTable();
-    }, [cluster, file, refreshTable, store]);
+    }, [cluster, file]);
 
     //  Command Bar  ---
     const buttons = React.useMemo((): ICommandBarItemProps[] => [

esp/src/src-react/components/Helpers.tsx

@@ -1,14 +1,11 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import * as domClass from "dojo/dom-class";
 import * as ESPRequest from "src/ESPRequest";
-import { Memory } from "src/Memory";
 import * as Utility from "src/Utility";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { HelperRow, useWorkunitHelpers } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 import { selector } from "./DojoGrid";
 
@@ -102,11 +99,12 @@ export const Helpers: React.FunctionComponent<HelpersProps> = ({
 
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
     const [helpers, refreshData] = useWorkunitHelpers(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Memory("id"));
     const [Grid, selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "id",
         filename: "helpers",
         columns: {
             sel: selector({
@@ -205,13 +203,13 @@ export const Helpers: React.FunctionComponent<HelpersProps> = ({
     }, [selection]);
 
     React.useEffect(() => {
-        store.setData(helpers);
-    }, [store, helpers]);
+        setData(helpers);
+    }, [helpers]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/ProtectedBy.tsx

@@ -1,10 +1,7 @@
 import * as React from "react";
 import { CommandBar, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import * as Observable from "dojo/store/Observable";
-import { Memory } from "src/Memory";
 import nlsHPCC from "src/nlsHPCC";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { useFile } from "../hooks/file";
 import { HolyGrail } from "../layouts/HolyGrail";
 
@@ -19,16 +16,17 @@ export const ProtectedBy: React.FunctionComponent<ProtectedByProps> = ({
 }) => {
 
     const [file, , , refreshData] = useFile(cluster, logicalFile);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("Owner")));
-    const [Grid, _selection, refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, _selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "Owner",
         sort: [{ attribute: "Owner", "descending": false }],
         filename: "protectedBy",
         columns: {
-            Owner: { label: nlsHPCC.Owner, sortable: false },
-            Modified: { label: nlsHPCC.Modified, sortable: false },
+            Owner: { label: nlsHPCC.Owner, width: 320 },
+            Modified: { label: nlsHPCC.Modified, width: 320 },
         }
     });
 
@@ -36,15 +34,14 @@ export const ProtectedBy: React.FunctionComponent<ProtectedByProps> = ({
         const results = file?.ProtectList?.DFUFileProtect;
 
         if (results) {
-            store.setData(file?.ProtectList?.DFUFileProtect?.map(row => {
+            setData(file?.ProtectList?.DFUFileProtect?.map(row => {
                 return {
                     Owner: row.Owner,
                     Modified: row.Modified
                 };
             }));
-            refreshTable();
         }
-    }, [store, file?.ProtectList?.DFUFileProtect, refreshTable]);
+    }, [file?.ProtectList?.DFUFileProtect]);
 
     //  Command Bar  ---
     const buttons = React.useMemo((): ICommandBarItemProps[] => [

esp/src/src-react/components/Resources.tsx

@@ -1,11 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { AlphaNumSortMemory } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { useWorkunitResources } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 import { selector } from "./DojoGrid";
 
@@ -23,11 +20,13 @@ export const Resources: React.FunctionComponent<ResourcesProps> = ({
 
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
     const [resources, , , refreshData] = useWorkunitResources(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new AlphaNumSortMemory("DisplayPath", { Name: true, Value: true }));
     const [Grid, selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "DisplayPath",
+        alphaNumColumns: { Name: true, Value: true },
         sort: [{ attribute: "Wuid", "descending": true }],
         filename: "resources",
         columns: {
@@ -89,18 +88,18 @@ export const Resources: React.FunctionComponent<ResourcesProps> = ({
     }, [selection]);
 
     React.useEffect(() => {
-        store.setData(resources.filter((row, idx) => idx > 0).map(row => {
+        setData(resources.filter((row, idx) => idx > 0).map(row => {
             return {
                 URL: row,
                 DisplayPath: row.substring(`res/${wuid}/`.length)
             };
         }));
-    }, [store, resources, wuid]);
+    }, [resources, wuid]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/Results.tsx

@@ -1,11 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { AlphaNumSortMemory } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { useWorkunitResults } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 import { selector } from "./DojoGrid";
 
@@ -23,11 +20,13 @@ export const Results: React.FunctionComponent<ResultsProps> = ({
 
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
     const [results, , , refreshData] = useWorkunitResults(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new AlphaNumSortMemory("__hpcc_id", { Name: true, Value: true }));
     const [Grid, selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "__hpcc_id",
+        alphaNumColumns: { Name: true, Value: true },
         sort: [{ attribute: "Wuid", "descending": true }],
         filename: "results",
         columns: {
@@ -108,7 +107,7 @@ export const Results: React.FunctionComponent<ResultsProps> = ({
     }, [selection]);
 
     React.useEffect(() => {
-        store.setData(results.map(row => {
+        setData(results.map(row => {
             const tmp: any = row.ResultViews;
             return {
                 __hpcc_id: row.Name,
@@ -120,12 +119,12 @@ export const Results: React.FunctionComponent<ResultsProps> = ({
                 Sequence: row.Sequence
             };
         }));
-    }, [store, results]);
+    }, [results]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/SourceFiles.tsx

@@ -1,15 +1,12 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Image, Link } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Image, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import * as domClass from "dojo/dom-class";
-import { AlphaNumSortMemory } from "src/Memory";
 import * as Utility from "src/Utility";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { useWorkunitSourceFiles } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
-import { selector, tree } from "./DojoGrid";
+import { selector } from "./DojoGrid";
 
 const defaultUIState = {
     hasSelection: false
@@ -19,28 +16,19 @@ interface SourceFilesProps {
     wuid: string;
 }
 
-class TreeStore extends AlphaNumSortMemory {
-
-    mayHaveChildren(item) {
-        return item.IsSuperFile;
-    }
-
-    getChildren(parent, options) {
-        return this.query({ __hpcc_parentName: parent.Name }, options);
-    }
-}
-
 export const SourceFiles: React.FunctionComponent<SourceFilesProps> = ({
     wuid
 }) => {
 
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
     const [sourceFiles, , , refreshData] = useWorkunitSourceFiles(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new TreeStore("Name", { Name: true, Value: true }));
     const [Grid, selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "Name",
+        alphaNumColumns: { Name: true, Value: true },
         sort: [{ attribute: "Name", "descending": false }],
         query: { __hpcc_parentName: "" },
         filename: "sourceFiles",
@@ -49,16 +37,16 @@ export const SourceFiles: React.FunctionComponent<SourceFilesProps> = ({
                 width: 27,
                 selectorType: "checkbox"
             }),
-            Name: tree({
+            Name: {
                 label: "Name", sortable: true,
                 formatter: function (Name, row) {
                     return <>
-                        <Image src={Utility.getImageURL(row.IsSuperFile ? "folder_table.png" : "file.png")} className='iconAlign' />
+                        <Image src={Utility.getImageURL(row.IsSuperFile ? "folder_table.png" : "file.png")} />
                         &nbsp;
                         <Link href={`#/files/${row.FileCluster}/${Name}`}>{Name}</Link>
                     </>;
                 }
-            }),
+            },
             FileCluster: { label: nlsHPCC.FileCluster, width: 300, sortable: false },
             Count: {
                 label: nlsHPCC.Usage, width: 72, sortable: true,
@@ -103,13 +91,13 @@ export const SourceFiles: React.FunctionComponent<SourceFilesProps> = ({
     }, [selection]);
 
     React.useEffect(() => {
-        store.setData(sourceFiles);
-    }, [store, sourceFiles]);
+        setData(sourceFiles);
+    }, [sourceFiles]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/SubFiles.tsx

@@ -1,15 +1,11 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { HolyGrail } from "../layouts/HolyGrail";
+import { CommandBar, ContextualMenuItemType, FontIcon, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import * as ESPLogicalFile from "src/ESPLogicalFile";
 import * as WsDfu from "src/WsDfu";
 import * as Utility from "src/Utility";
 import { useConfirm } from "../hooks/confirm";
 import { useFile } from "../hooks/file";
-import { useGrid } from "../hooks/grid";
-import * as Observable from "dojo/store/Observable";
-import { Memory } from "src/Memory";
+import { useFluentGrid } from "../hooks/grid";
 import { ShortVerticalDivider } from "./Common";
 import { selector } from "./DojoGrid";
 import { pushUrl } from "../util/history";
@@ -31,11 +27,13 @@ export const SubFiles: React.FunctionComponent<SubFilesProps> = ({
 
     const [file, , , refresh] = useFile(cluster, logicalFile);
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("Name")));
-    const [Grid, selection, refreshTable] = useGrid({
-        store: store,
+    const [Grid, selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "Name",
+        alphaNumColumns: { RecordCount: true, Totalsize: true },
         query: {},
         sort: [{ attribute: "Modified", "descending": true }],
         filename: "subfiles",
@@ -43,66 +41,60 @@ export const SubFiles: React.FunctionComponent<SubFilesProps> = ({
             sel: selector({ width: 27, selectorType: "checkbox" }),
             IsCompressed: {
                 width: 25, sortable: false,
-                renderHeaderCell: function (node) {
-                    node.innerHTML = Utility.getImageHTML("compressed.png", nlsHPCC.Compressed);
-                },
+                headerIcon: "ZipFolder",
+                headerTooltip: nlsHPCC.Compressed,
                 formatter: function (compressed) {
                     if (compressed === true) {
-                        return Utility.getImageHTML("compressed.png");
+                        return <FontIcon iconName="zipFolder" />;
                     }
-                    return "";
+                    return <></>;
                 }
             },
             IsKeyFile: {
                 width: 25, sortable: false,
-                renderHeaderCell: function (node) {
-                    node.innerHTML = Utility.getImageHTML("index.png", nlsHPCC.Index);
-                },
+                headerIcon: "Permissions",
+                headerTooltip: nlsHPCC.Index,
                 formatter: function (keyfile, row) {
                     if (row.ContentType === "key") {
-                        return Utility.getImageHTML("index.png");
+                        return <FontIcon iconName="Permissions" />;
                     }
-                    return "";
+                    return <></>;
                 }
             },
             isSuperfile: {
                 width: 25, sortable: false,
-                renderHeaderCell: function (node) {
-                    node.innerHTML = Utility.getImageHTML("superfile.png", nlsHPCC.Superfile);
-                },
+                headerIcon: "Folder",
+                headerTooltip: nlsHPCC.Superfile,
                 formatter: function (superfile) {
                     if (superfile === true) {
-                        return Utility.getImageHTML("superfile.png");
+                        return <FontIcon iconName="Folder" />;
                     }
-                    return "";
+                    return <></>;
                 }
             },
             Name: {
                 label: nlsHPCC.LogicalName,
                 formatter: function (name, row) {
                     const url = "#/files/" + (row.NodeGroup ? row.NodeGroup + "/" : "") + name;
-                    return "<a href='" + url + "'>" + name + "</a>";
+                    return <Link href={url}>{name}</Link>;
                 }
             },
             Owner: { label: nlsHPCC.Owner, width: 72 },
             Description: { label: nlsHPCC.Description, width: 153 },
             RecordCount: {
                 label: nlsHPCC.Records, width: 72, sortable: false,
-                renderCell: function (object, value, node, options) {
-                    node.innerText = Utility.valueCleanUp(value);
+                formatter: function (recordCount, row) {
+                    return Utility.valueCleanUp(recordCount);
                 },
             },
             Totalsize: {
                 label: nlsHPCC.Size, width: 72, sortable: false,
-                renderCell: function (object, value, node, options) {
-                    node.innerText = Utility.valueCleanUp(value);
+                formatter: function (totalSize, row) {
+                    return Utility.valueCleanUp(totalSize);
                 },
             },
             Parts: {
                 label: nlsHPCC.Parts, width: 45, sortable: false,
-                renderCell: function (object, value, node, options) {
-                    node.innerText = value;
-                },
             },
             Modified: { label: nlsHPCC.ModifiedUTCGMT, width: 155, sortable: false }
         }
@@ -130,10 +122,9 @@ export const SubFiles: React.FunctionComponent<SubFilesProps> = ({
             subfiles.push(logicalFile);
         });
         Promise.all(promises).then(logicalFiles => {
-            store.setData(subfiles);
-            refreshTable();
+            setData(subfiles);
         });
-    }, [store, file?.subfiles, refreshTable]);
+    }, [file?.subfiles]);
 
     const buttons = React.useMemo((): ICommandBarItemProps[] => [
         {
@@ -166,11 +157,12 @@ export const SubFiles: React.FunctionComponent<SubFilesProps> = ({
     }, [selection]);
 
     return <>
-        <HolyGrail
-            header={<CommandBar items={buttons} />}
-            main={<Grid />}
-        />
+        <ScrollablePane>
+            <Sticky>
+                <CommandBar items={buttons} farItems={copyButtons} />
+            </Sticky>
+            <Grid />
+        </ScrollablePane>
         <DeleteSubfilesConfirm />
     </>;
-
 };

esp/src/src-react/components/SuperFiles.tsx

@@ -1,12 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import * as Observable from "dojo/store/Observable";
-import { Memory } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Link, ScrollablePane, Sticky } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
-import { useGrid } from "../hooks/grid";
+import { useFluentGrid } from "../hooks/grid";
 import { useFile } from "../hooks/file";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 import { selector } from "./DojoGrid";
 
@@ -26,11 +22,12 @@ export const SuperFiles: React.FunctionComponent<SuperFilesProps> = ({
 
     const [file, , , refreshData] = useFile(cluster, logicalFile);
     const [uiState, setUIState] = React.useState({ ...defaultUIState });
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new Observable(new Memory("Name")));
-    const [Grid, selection, refreshTable, copyButtons] = useGrid({
-        store,
+    const [Grid, selection, copyButtons] = useFluentGrid({
+        data,
+        primaryID: "Name",
         sort: [{ attribute: "Name", "descending": false }],
         filename: "superFiles",
         columns: {
@@ -38,7 +35,13 @@ export const SuperFiles: React.FunctionComponent<SuperFilesProps> = ({
                 width: 27,
                 selectorType: "checkbox"
             }),
-            Name: { label: nlsHPCC.Name, sortable: true, },
+            Name: {
+                label: nlsHPCC.Name,
+                sortable: true,
+                formatter: function (name, row) {
+                    return <Link href={`#/files/${cluster}/${name}`}>{name}</Link>;
+                }
+            },
         }
     });
 
@@ -75,15 +78,14 @@ export const SuperFiles: React.FunctionComponent<SuperFilesProps> = ({
 
     React.useEffect(() => {
         if (file?.Superfiles?.DFULogicalFile) {
-            store?.setData(file?.Superfiles?.DFULogicalFile);
-            refreshTable();
+            setData(file?.Superfiles?.DFULogicalFile);
         }
-    }, [file, store, refreshTable]);
+    }, [file]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/Title.tsx

@@ -8,6 +8,7 @@ import * as WsAccount from "src/ws_account";
 import * as cookie from "dojo/cookie";
 
 import nlsHPCC from "src/nlsHPCC";
+import { useBanner } from "../hooks/banner";
 import { useECLWatchLogger } from "../hooks/logging";
 import { useGlobalStore } from "../hooks/store";
 import * as Utility from "src/Utility";
@@ -42,6 +43,9 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
     const [environmentTitle, setEnvironmentTitle] = useGlobalStore("HPCCPlatformWidget_Toolbar_Text", toolbarThemeDefaults.text);
     const [titlebarColor, setTitlebarColor] = useGlobalStore("HPCCPlatformWidget_Toolbar_Color", toolbarThemeDefaults.color);
 
+    const [showBannerConfig, setShowBannerConfig] = React.useState(false);
+    const [BannerMessageBar, BannerConfig] = useBanner({ showForm: showBannerConfig, setShowForm: setShowBannerConfig });
+
     const personaProps: IPersonaSharedProps = React.useMemo(() => {
         return {
             text: currentUser?.firstName + " " + currentUser?.lastName,
@@ -68,7 +72,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
             items: [
                 { key: "legacy", text: nlsHPCC.OpenLegacyECLWatch, href: "/esp/files/stub.htm" },
                 { key: "divider_0", itemType: ContextualMenuItemType.Divider },
-                { key: "banner", text: nlsHPCC.SetBanner },
+                { key: "banner", text: nlsHPCC.SetBanner, onClick: () => setShowBannerConfig(true) },
                 { key: "toolbar", text: nlsHPCC.SetToolbar, onClick: () => setShowTitlebarConfig(true) },
                 { key: "divider_1", itemType: ContextualMenuItemType.Divider },
                 { key: "docs", href: "https://hpccsystems.com/training/documentation/", text: nlsHPCC.Documentation, target: "_blank" },
@@ -143,6 +147,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
     }, [environmentTitle]);
 
     return <div style={{ backgroundColor: titlebarColor ?? theme.palette.themeLight }}>
+        <BannerMessageBar />
         <Stack horizontal verticalAlign="center" horizontalAlign="space-between">
             <Stack.Item align="center">
                 <Stack horizontal>
@@ -200,6 +205,7 @@ export const DevTitle: React.FunctionComponent<DevTitleProps> = ({
             environmentTitle={environmentTitle} setEnvironmentTitle={setEnvironmentTitle}
             titlebarColor={titlebarColor} setTitlebarColor={setTitlebarColor}
         />
+        <BannerConfig />
     </div>;
 };
 

esp/src/src-react/components/Variables.tsx

@@ -1,11 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { AlphaNumSortMemory } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, ScrollablePane, Sticky } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { useWorkunitVariables } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 
 interface VariablesProps {
@@ -17,11 +14,13 @@ export const Variables: React.FunctionComponent<VariablesProps> = ({
 }) => {
 
     const [variables, , , refreshData] = useWorkunitVariables(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new AlphaNumSortMemory("__hpcc_id", { Name: true, Value: true }));
     const [Grid, _selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "__hpcc_id",
+        alphaNumColumns: { Name: true, Value: true },
         sort: [{ attribute: "Wuid", "descending": true }],
         filename: "variables",
         columns: {
@@ -32,13 +31,13 @@ export const Variables: React.FunctionComponent<VariablesProps> = ({
     });
 
     React.useEffect(() => {
-        store.setData(variables.map((row, idx) => {
+        setData(variables.map((row, idx) => {
             return {
                 __hpcc_id: idx,
                 ...row
             };
         }));
-    }, [store, variables]);
+    }, [variables]);
 
     //  Command Bar  ---
     const buttons = React.useMemo((): ICommandBarItemProps[] => [
@@ -49,10 +48,10 @@ export const Variables: React.FunctionComponent<VariablesProps> = ({
         { key: "divider_1", itemType: ContextualMenuItemType.Divider, onRender: () => <ShortVerticalDivider /> },
     ], [refreshData]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/components/Workflows.tsx

@@ -1,11 +1,8 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react";
-import { useConst } from "@fluentui/react-hooks";
-import { AlphaNumSortMemory } from "src/Memory";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, ScrollablePane, Sticky } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
 import { useFluentGrid } from "../hooks/grid";
 import { useWorkunitWorkflows } from "../hooks/workunit";
-import { HolyGrail } from "../layouts/HolyGrail";
 import { ShortVerticalDivider } from "./Common";
 
 interface WorkflowsProps {
@@ -17,11 +14,13 @@ export const Workflows: React.FunctionComponent<WorkflowsProps> = ({
 }) => {
 
     const [workflows, , refreshWorkflow] = useWorkunitWorkflows(wuid);
+    const [data, setData] = React.useState<any[]>([]);
 
     //  Grid ---
-    const store = useConst(new AlphaNumSortMemory("__hpcc_id", { Name: true, Value: true }));
     const [Grid, _selection, copyButtons] = useFluentGrid({
-        store,
+        data,
+        primaryID: "__hpcc_id",
+        alphaNumColumns: { Name: true, Value: true },
         sort: [{ attribute: "Wuid", "descending": true }],
         filename: "workflows",
         columns: {
@@ -49,13 +48,13 @@ export const Workflows: React.FunctionComponent<WorkflowsProps> = ({
     });
 
     React.useEffect(() => {
-        store.setData(workflows.map(row => {
+        setData(workflows.map(row => {
             return {
                 ...row,
                 __hpcc_id: row.WFID
             };
         }));
-    }, [store, workflows]);
+    }, [workflows]);
 
     //  Command Bar  ---
     const buttons = React.useMemo((): ICommandBarItemProps[] => [
@@ -68,10 +67,10 @@ export const Workflows: React.FunctionComponent<WorkflowsProps> = ({
         { key: "divider_1", itemType: ContextualMenuItemType.Divider, onRender: () => <ShortVerticalDivider /> },
     ], [refreshWorkflow]);
 
-    return <HolyGrail
-        header={<CommandBar items={buttons} farItems={copyButtons} />}
-        main={
-            <Grid />
-        }
-    />;
+    return <ScrollablePane>
+        <Sticky>
+            <CommandBar items={buttons} farItems={copyButtons} />
+        </Sticky>
+        <Grid />
+    </ScrollablePane>;
 };

esp/src/src-react/hooks/activity.ts

@@ -0,0 +1,30 @@
+import * as React from "react";
+import { Activity } from "@hpcc-js/comms";
+import { useCounter } from "./workunit";
+
+export function useActivity(): [Activity, number, () => void] {
+
+    const [activity, setActivity] = React.useState<Activity>();
+    const [lastUpdate, setLastUpdate] = React.useState(Date.now());
+    const [count, increment] = useCounter();
+
+    React.useEffect(() => {
+        const activity = Activity.attach({ baseUrl: "" });
+        let active = true;
+        let handle;
+        activity.lazyRefresh().then(() => {
+            if (active) {
+                setActivity(activity);
+                handle = activity.watch(() => {
+                    setLastUpdate(Date.now());
+                });
+            }
+        });
+        return () => {
+            active = false;
+            handle.release();
+        }
+    }, [count]);
+
+    return [activity, lastUpdate, increment];
+}

esp/src/src-react/hooks/banner.tsx

@@ -0,0 +1,194 @@
+import * as React from "react";
+import { Checkbox, ColorPicker, DefaultButton, getColorFromString, IColor, Label, MessageBar, MessageBarType, PrimaryButton, Stack, TextField } from "@fluentui/react";
+import { useForm, Controller } from "react-hook-form";
+import nlsHPCC from "src/nlsHPCC";
+import { useActivity } from "./activity";
+import { MessageBox } from "../layouts/MessageBox";
+
+interface BannerConfigValues {
+    BannerAction: number;
+    BannerContent: string;
+    BannerColor: string;
+    BannerSize: string;
+    BannerScroll: string;
+}
+
+const defaultValues: BannerConfigValues = {
+    BannerAction: 0,
+    BannerContent: "",
+    BannerColor: "",
+    BannerSize: "",
+    BannerScroll: ""
+};
+
+interface useBannerProps {
+    showForm: boolean;
+    setShowForm: (_: boolean) => void;
+}
+
+const white = getColorFromString("#ffffff");
+
+export function useBanner({ showForm, setShowForm }: useBannerProps): [React.FunctionComponent, React.FunctionComponent] {
+
+    const [activity] = useActivity();
+
+    const [bannerColor, setBannerColor] = React.useState(activity?.BannerColor || "black");
+    const [bannerMessage, setBannerMessage] = React.useState(activity?.BannerContent || "");
+    const [bannerSize, setBannerSize] = React.useState(activity?.BannerSize || "16");
+    const [showBanner, setShowBanner] = React.useState(activity?.ShowBanner == 1 || false);
+
+    const { handleSubmit, control, reset } = useForm<BannerConfigValues>({ defaultValues });
+    const [color, setColor] = React.useState(white);
+    const updateColor = React.useCallback((evt: any, colorObj: IColor) => setColor(colorObj), []);
+
+    const closeForm = React.useCallback(() => {
+        setShowForm(false);
+    }, [setShowForm]);
+
+    const onSubmit = React.useCallback(() => {
+        handleSubmit(
+            (data, evt) => {
+                const request: any = data;
+                request.BannerColor = color.str;
+                request.BannerAction = request.BannerAction === true ? "1" : "0";
+                setBannerColor(request.BannerColor);
+                setBannerMessage(request.BannerContent);
+                setBannerSize(request.BannerSize);
+                setShowBanner(request.BannerAction == 1);
+                activity.setBanner(request);
+                closeForm();
+            },
+        )();
+    }, [activity, closeForm, color, handleSubmit]);
+
+    React.useEffect(() => {
+        if (!activity?.BannerColor) return;
+        setColor(getColorFromString(activity?.BannerColor));
+        const values = {
+            BannerAction: activity?.ShowBanner || 0,
+            BannerContent: activity?.BannerContent || "",
+            BannerScroll: activity?.BannerScroll || "",
+            BannerSize: activity?.BannerSize || ""
+        };
+        reset(values);
+    }, [activity?.BannerColor, activity?.BannerContent, activity?.BannerScroll, activity?.BannerSize, activity?.ShowBanner, reset]);
+
+    const BannerConfig = React.useMemo(() => () => {
+        return <MessageBox show={showForm} setShow={closeForm} title={nlsHPCC.SetBanner} minWidth={680}
+            footer={<>
+                <PrimaryButton text={nlsHPCC.OK} onClick={handleSubmit(onSubmit)} />
+                <DefaultButton text={nlsHPCC.Cancel} onClick={closeForm} />
+            </>}>
+            <Stack horizontal horizontalAlign="space-between">
+                <Stack.Item grow={2}>
+                    <Controller
+                        control={control} name="BannerAction"
+                        render={({
+                            field: { onChange, name: fieldName, value }
+                        }) => <Controller
+                                control={control} name={fieldName}
+                                render={({
+                                    field: { onChange, name: fieldName, value }
+                                }) => <Checkbox name={fieldName} checked={value == 1} onChange={onChange} label={nlsHPCC.Enable} />}
+                            />
+                        }
+                    />
+                    <Controller
+                        control={control} name="BannerContent"
+                        render={({
+                            field: { onChange, name: fieldName, value },
+                            fieldState: { error }
+                        }) => <TextField
+                                name={fieldName}
+                                onChange={onChange}
+                                required={true}
+                                multiline
+                                autoAdjustHeight
+                                label={nlsHPCC.BannerMessage}
+                                value={value}
+                                errorMessage={error && error?.message}
+                            />}
+                        rules={{
+                            required: nlsHPCC.ValidationErrorRequired
+                        }}
+                    />
+                    <Controller
+                        control={control} name="BannerSize"
+                        render={({
+                            field: { onChange, name: fieldName, value },
+                            fieldState: { error }
+                        }) => <TextField
+                                name={fieldName}
+                                onChange={onChange}
+                                label={nlsHPCC.BannerSize}
+                                value={value}
+                                errorMessage={error && error?.message}
+                            />}
+                        rules={{
+                            pattern: {
+                                value: /^[0-9]+$/i,
+                                message: nlsHPCC.ValidationErrorEnterNumber
+                            }
+                        }}
+                    />
+                    <Controller
+                        control={control} name="BannerScroll"
+                        render={({
+                            field: { onChange, name: fieldName, value },
+                            fieldState: { error }
+                        }) => <TextField
+                                name={fieldName}
+                                onChange={onChange}
+                                label={nlsHPCC.BannerScroll}
+                                value={value}
+                                errorMessage={error && error?.message}
+                            />}
+                        rules={{
+                            pattern: {
+                                value: /^[0-9]+$/i,
+                                message: nlsHPCC.ValidationErrorEnterNumber
+                            }
+                        }}
+                    />
+                </Stack.Item>
+                <Stack.Item>
+                    <Label>{nlsHPCC.BannerColor}</Label>
+                    <ColorPicker
+                        onChange={updateColor}
+                        color={color}
+                    />
+                </Stack.Item>
+            </Stack>
+        </MessageBox>;
+    }, [closeForm, color, control, handleSubmit, onSubmit, showForm, updateColor]);
+
+    React.useEffect(() => {
+        setShowBanner(activity?.ShowBanner == 1);
+        setBannerMessage(activity?.BannerContent || "");
+        setBannerColor(activity?.BannerColor || "black");
+        setBannerSize(activity?.BannerSize || "16");
+    }, [activity?.BannerContent, activity?.ShowBanner, activity?.BannerColor, activity?.BannerSize]);
+
+    const BannerMessageBar = React.useMemo(() => () => {
+        return showBanner &&
+            <MessageBar
+                messageBarType={MessageBarType.warning}
+                onDismiss={() => setShowBanner(false)}
+                dismissButtonAriaLabel="Close"
+                isMultiline={false}
+                truncated={true}
+                overflowButtonAriaLabel="See More"
+                style={{
+                    color: bannerColor,
+                    fontSize: `${bannerSize}px`,
+                    lineHeight: `${bannerSize}px`
+                }}
+            >
+                {bannerMessage}
+            </MessageBar>
+            ;
+    }, [bannerColor, bannerMessage, bannerSize, showBanner]);
+
+    return [BannerMessageBar, BannerConfig];
+
+}

esp/src/src-react/hooks/grid.tsx

@@ -1,6 +1,7 @@
 import * as React from "react";
-import { DetailsList, DetailsListLayoutMode, IColumn, ICommandBarItemProps, Selection } from "@fluentui/react";
+import { DetailsList, DetailsListLayoutMode, IColumn, ICommandBarItemProps, IDetailsHeaderProps, Selection, TooltipHost } from "@fluentui/react";
 import { useConst } from "@fluentui/react-hooks";
+import { AlphaNumSortMemory } from "src/Memory";
 import { createCopyDownloadSelection } from "../components/Common";
 import { DojoGrid } from "../components/DojoGrid";
 import { useDeepCallback, useDeepEffect } from "./deepHooks";
@@ -72,24 +73,58 @@ function columnsAdapter(columns, sorted: Sorted): IColumn[] {
                 isResizable: true,
                 isSorted: key == sorted.column,
                 isSortedDescending: key == sorted.column && sorted.descending,
+                iconName: column.headerIcon,
+                isIconOnly: !!column.headerIcon,
+                data: column
             } as IColumn);
         }
     }
     return retVal;
 }
 
-export function useFluentGrid({ store, query = {}, sort = [], columns, getSelected, filename }: useGridProps): [React.FunctionComponent, any[], ICommandBarItemProps[]] {
+export interface useFluentGrid2Props {
+    data: any[],
+    primaryID: string,
+    alphaNumColumns?: { [id: string]: boolean },
+    query?: object,
+    sort?: object[],
+    columns: object,
+    getSelected?: () => any[],
+    filename: string
+}
+
+export function useFluentGrid({ data, primaryID, alphaNumColumns = {}, query = {}, sort = [], columns, getSelected, filename }: useFluentGrid2Props): [React.FunctionComponent, any[], ICommandBarItemProps[]] {
 
+    const constStore = useConst(new AlphaNumSortMemory(primaryID, alphaNumColumns));
     const constQuery = useConst({ ...query });
     const constColumns = useConst({ ...columns });
     const [sorted, setSorted] = React.useState<Sorted>({ column: "", descending: false });
     const [selection, setSelection] = React.useState([]);
+    const [items, setItems] = React.useState<any[]>([]);
+
+    const refreshTable = React.useCallback(() => {
+        const sort = sorted.column ? [{ attribute: sorted.column, descending: sorted.descending }] : undefined;
+        constStore.query(constQuery, { sort }).then(items => {
+            setItems(items);
+        });
+    }, [constQuery, constStore, sorted.column, sorted.descending]);
+
+    React.useEffect(() => {
+        refreshTable();
+    }, [refreshTable]);
+
+    React.useEffect(() => {
+        constStore.setData(data);
+        refreshTable();
+    }, [constStore, data, refreshTable]);
 
     const fluentColumns: IColumn[] = React.useMemo(() => {
         return columnsAdapter(constColumns, sorted);
     }, [constColumns, sorted]);
 
     const onColumnClick = React.useCallback((event: React.MouseEvent<HTMLElement>, column: IColumn) => {
+        if (constColumns[column.key]?.sortable === false) return;
+
         let sorted = column.isSorted;
         let isSortedDescending: boolean = column.isSortedDescending;
         if (!sorted) {
@@ -105,16 +140,14 @@ export function useFluentGrid({ store, query = {}, sort = [], columns, getSelect
             column: sorted ? column.key : "",
             descending: sorted ? isSortedDescending : false
         });
-    }, []);
-
-    const [items, setItems] = React.useState<any[]>([]);
+    }, [constColumns]);
 
     React.useEffect(() => {
         const sort = sorted.column ? [{ attribute: sorted.column, descending: sorted.descending }] : undefined;
-        store.query(constQuery, { sort }).then(items => {
+        constStore.query(constQuery, { sort }).then(items => {
             setItems(items);
         });
-    }, [constQuery, sorted.column, sorted.descending, store, store.dataVersion]);
+    }, [constQuery, constStore, sorted.column, sorted.descending]);
 
     const selectionHandler = useConst(new Selection({
         onSelectionChanged: () => {
@@ -122,6 +155,16 @@ export function useFluentGrid({ store, query = {}, sort = [], columns, getSelect
         }
     }));
 
+    const renderDetailsHeader = React.useCallback((props: IDetailsHeaderProps, defaultRender?: any) => {
+        return defaultRender({
+            ...props,
+            onRenderColumnHeaderTooltip: (tooltipHostProps) => {
+                return <TooltipHost {...tooltipHostProps} content={tooltipHostProps?.column?.data?.headerTooltip ?? ""} />;
+            },
+            styles: { root: { paddingTop: 1 } }
+        });
+    }, []);
+
     const renderItemColumn = React.useCallback((item: any, index: number, column: IColumn) => {
         if (constColumns[column.key].formatter) {
             return <span style={{ display: "flex" }}>{constColumns[column.key].formatter(item[column.key], item)}</span>;
@@ -140,7 +183,9 @@ export function useFluentGrid({ store, query = {}, sort = [], columns, getSelect
         selectionPreservedOnEmptyClick={true}
         onItemInvoked={this._onItemInvoked}
         onColumnHeaderClick={onColumnClick}
-    />, [fluentColumns, items, onColumnClick, renderItemColumn, selectionHandler]);
+        onRenderDetailsHeader={renderDetailsHeader}
+
+    />, [fluentColumns, items, onColumnClick, renderDetailsHeader, renderItemColumn, selectionHandler]);
 
     const copyButtons = React.useMemo((): ICommandBarItemProps[] => [
         ...createCopyDownloadSelection(constColumns, selection, `${filename}.csv`)

esp/src/webpack.config.js

@@ -95,7 +95,11 @@ module.exports = function (env) {
         resolve: {
             alias: {
                 "clipboard": path.resolve(__dirname, "node_modules/clipboard/dist/clipboard")
-            }
+            },
+            // WebPack >= v5
+            // fallback: {
+            //     "@hpcc-js": path.resolve(__dirname, "../../../hpcc-js/packages")
+            // },
         },
         plugins: plugins,
         resolveLoader: {

roxie/udplib/CMakeLists.txt

@@ -28,4 +28,5 @@ project (AllProjects)
 
 include ( udplib.cmake)
 include ( udptransport.cmake)
+include ( udpsim.cmake)
 

roxie/udplib/udplib.hpp

@@ -161,6 +161,7 @@ extern UDPLIB_API RelaxedAtomic<unsigned> unwantedDiscarded;
 
 extern UDPLIB_API bool udpTraceFlow;
 extern UDPLIB_API bool udpTraceTimeouts;
+
 extern UDPLIB_API unsigned udpTraceLevel;
 extern UDPLIB_API unsigned udpOutQsPriority;
 extern UDPLIB_API void queryMemoryPoolStats(StringBuffer &memStats);

roxie/udplib/udpsha.cpp

@@ -41,6 +41,12 @@ unsigned udpFlowSocketsSize = 131072;
 unsigned udpLocalWriteSocketSize = 1024000;
 unsigned udpStatsReportInterval = 60000;
 
+#ifdef TEST_DROPPED_PACKETS
+bool udpDropDataPackets = false;
+unsigned udpDropFlowPackets[flowType::max_flow_cmd] = {};
+unsigned flowPacketsSent[flowType::max_flow_cmd] = {};
+#endif
+
 unsigned multicastTTL = 1;
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)
@@ -673,7 +679,6 @@ fake read socket that
 #ifdef SOCKET_SIMULATION
 bool isUdpTestMode = false;
 
-
 CSimulatedQueueWriteSocket* CSimulatedQueueWriteSocket::udp_connect(const SocketEndpoint &ep)
 {
     return new CSimulatedQueueWriteSocket(ep);
@@ -844,83 +849,4 @@ void CSimulatedUdpWriteSocket::close()
     realSocket->close();
 }
 
-
-
-//-----------------------------------------------------------------------------------------------------
-
-#ifdef _USE_CPPUNIT
-
-class SimulatedUdpStressTest : public CppUnit::TestFixture
-{
-    CPPUNIT_TEST_SUITE(SimulatedUdpStressTest);
-    CPPUNIT_TEST(simulateTraffic);
-    CPPUNIT_TEST_SUITE_END();
-
-    Owned<IDataBufferManager> dbm;
-    bool initialized = false;
-
-    void testInit()
-    {
-        if (!initialized)
-        {
-            udpTraceLevel = 1;
-            udpTraceTimeouts = true;
-            udpResendLostPackets = true;
-            udpRequestToSendTimeout = 10000;
-            udpRequestToSendAckTimeout = 10000;
-            udpMaxPendingPermits = 1;
-            udpTraceFlow = 0;
-            isUdpTestMode = true;
-            roxiemem::setTotalMemoryLimit(false, false, false, 20*1024*1024, 0, NULL, NULL);
-            dbm.setown(roxiemem::createDataBufferManager(roxiemem::DATA_ALIGNMENT_SIZE));
-            initialized = true;
-        }
-    }
-
-    void simulateTraffic()
-    {
-        constexpr unsigned numReceiveSlots = 100;
-        constexpr unsigned maxSlotsPerClient = 100;
-        constexpr unsigned maxSendQueueSize = 100;
-        try
-        {
-            testInit();
-            myNode.setIp(IpAddress("1.2.3.4"));
-            Owned<IReceiveManager> rm = createReceiveManager(CCD_SERVER_FLOW_PORT, CCD_DATA_PORT, CCD_CLIENT_FLOW_PORT, numReceiveSlots, maxSlotsPerClient, false);
-            unsigned begin = msTick();
-            printf("Start test\n");
-            asyncFor(20, 20, [](unsigned i)
-            {
-                unsigned header = 0;
-                IpAddress pretendIP(VStringBuffer("8.8.8.%d", i));
-                // Note - this is assuming we send flow on the data port (that option defaults true in roxie too)
-                Owned<ISendManager> sm = createSendManager(CCD_DATA_PORT, CCD_DATA_PORT, CCD_CLIENT_FLOW_PORT, maxSendQueueSize, 3, pretendIP, nullptr, false);
-                Owned<IMessagePacker> mp = sm->createMessagePacker(0, 0, &header, sizeof(header), myNode, 0);
-                for (unsigned i = 0; i < 10000; i++)
-                {
-                    void *buf = mp->getBuffer(500, false);
-                    memset(buf, i, 500);
-                    mp->putBuffer(buf, 500, false);
-                }
-                mp->flush();
-
-                //wait until all the packets have been sent and acknowledged
-                while(!sm->allDone())
-                    Sleep(50);
-            });
-            printf("End test %u\n", msTick() - begin);
-        }
-        catch (IException * e)
-        {
-            StringBuffer msg;
-            printf("Exception: %s\n", e->errorMessage(msg).str());
-            throw;
-        }
-    }
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION( SimulatedUdpStressTest );
-CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( SimulatedUdpStressTest, "SimulatedUdpStressTest" );
-
-#endif
 #endif

roxie/udplib/udpsha.hpp

@@ -202,7 +202,7 @@ public:
 
 class flowType {
 public:
-    enum flowCmd : unsigned short { ok_to_send, request_received, request_to_send, send_completed, request_to_send_more };
+    enum flowCmd : unsigned short { ok_to_send, request_received, request_to_send, send_completed, request_to_send_more, max_flow_cmd };
     static const char *name(flowCmd m)
     {
         switch (m)
@@ -267,7 +267,17 @@ inline bool checkTraceLevel(unsigned category, unsigned level)
 #define SOCKET_SIMULATION_UDP
 
 #ifdef SOCKET_SIMULATION
-extern bool isUdpTestMode;
+#ifdef _DEBUG
+#define TEST_DROPPED_PACKETS
+#endif
+
+#ifdef TEST_DROPPED_PACKETS
+extern UDPLIB_API bool udpDropDataPackets;
+extern UDPLIB_API unsigned udpDropFlowPackets[flowType::max_flow_cmd];
+extern unsigned flowPacketsSent[flowType::max_flow_cmd];
+#endif
+
+extern UDPLIB_API bool isUdpTestMode;
 
 class CSocketSimulator : public CInterfaceOf<ISocket>
 {
@@ -288,8 +298,9 @@ private:
     virtual bool set_nonblock(bool on) override { UNIMPLEMENTED; }
     virtual bool set_nagle(bool on) override { UNIMPLEMENTED; }
     virtual void set_linger(int lingersecs) override { UNIMPLEMENTED; }
-    virtual void  cancel_accept() override { UNIMPLEMENTED; }
-    virtual void  shutdown(unsigned mode=SHUTDOWN_READWRITE) override { UNIMPLEMENTED; }
+    virtual void cancel_accept() override { UNIMPLEMENTED; }
+    virtual void shutdown(unsigned mode) override { UNIMPLEMENTED; }
+    virtual void shutdownNoThrow(unsigned mode) override { UNIMPLEMENTED; }
     virtual int name(char *name,size32_t namemax) override { UNIMPLEMENTED; }
     virtual int peer_name(char *name,size32_t namemax) override { UNIMPLEMENTED; }
     virtual SocketEndpoint &getPeerEndpoint(SocketEndpoint &ep) override { UNIMPLEMENTED; }
@@ -378,7 +389,8 @@ public:
                          unsigned timeout) override;
     virtual int wait_read(unsigned timeout) override;
     virtual void close() override {}
-
+    virtual void  shutdown(unsigned mode) override { }
+    virtual void  shutdownNoThrow(unsigned mode) override{ }
 };
 
 class CSimulatedQueueWriteSocket : public CSocketSimulator
@@ -395,6 +407,8 @@ public:
 
 class CSimulatedUdpSocket : public CSocketSimulator
 {
+    virtual void  shutdown(unsigned mode) override { realSocket->shutdown(mode); }
+    virtual void  shutdownNoThrow(unsigned mode) override{ realSocket->shutdownNoThrow(mode); }
 protected:
     Owned<ISocket> realSocket;
 };

roxie/udplib/udpsim.cmake

@@ -0,0 +1,48 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: udptransport 
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for udpsim
+#####################################################
+
+
+project( udptransport ) 
+
+set (    SRCS 
+         udpsim.cpp 
+    )
+
+include_directories ( 
+         ./../../roxie/roxiemem 
+         ./../../system/include 
+         ./../../system/jlib 
+         ./../../roxie/ccd 
+    )
+
+HPCC_ADD_EXECUTABLE ( udpsim ${SRCS} )
+
+#We don't currently ship this - it's for developer use only
+#install ( TARGETS udpsim RUNTIME DESTINATION ${EXEC_DIR} )  
+
+target_link_libraries ( udpsim
+         jlib
+         roxiemem
+         udplib 
+    )

roxie/udplib/udpsim.cpp

@@ -0,0 +1,210 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2021 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "udplib.hpp"
+#include "udpsha.hpp"
+#include "udptrs.hpp"
+#include "udpipmap.hpp"
+#include "roxiemem.hpp"
+#include "jptree.hpp"
+#include "portlist.h"
+
+using roxiemem::DataBuffer;
+using roxiemem::IDataBufferManager;
+
+#ifdef SOCKET_SIMULATION
+
+Owned<IDataBufferManager> dbm;
+
+unsigned numThreads = 20;
+
+static constexpr const char * defaultYaml = R"!!(
+version: "1.0"
+udpsim:
+  dropDataPackets: false
+  dropOkToSendPackets: 0
+  dropRequestReceivedPackets: 0
+  dropRequestToSendPackets: 0
+  dropRequestToSendMorePackets: 0
+  dropSendCompletedPackets: 0
+  help: false
+  numThreads: 20
+  outputconfig: false
+  udpTraceLevel: 1
+  udpTraceTimeouts: true
+  udpResendLostPackets: true
+  udpRequestToSendTimeout: 1000
+  udpRequestToSendAckTimeout: 1000
+  udpMaxPendingPermits: 1
+  udpTraceFlow: false
+)!!";
+
+bool isNumeric(const char *str)
+{
+    while (*str)
+    {
+        if (!isdigit(*str))
+            return false;
+        str++;
+    }
+    return true;
+}
+
+bool isBoolean(const char *str)
+{
+    return streq(str, "true") || streq(str, "false");
+}
+
+void usage()
+{
+    printf("USAGE: udpsim [options]\n");
+    printf("Options are:\n");
+    Owned<IPropertyTree> defaults = createPTreeFromYAMLString(defaultYaml);
+    IPropertyTree * allowed = defaults->queryPropTree("udpsim");
+    Owned<IAttributeIterator> aiter = allowed->getAttributes();
+    ForEach(*aiter)
+    {
+        printf("  --%s", aiter->queryName()+1);
+        if (isBoolean(aiter->queryValue()))
+            printf("[=0|1]\n");
+        else
+            printf("=nn\n");
+    }
+    ExitModuleObjects();
+    releaseAtoms();
+    exit(2);
+}
+
+void initOptions(int argc, const char **argv)
+{
+    Owned<IPropertyTree> defaults = createPTreeFromYAMLString(defaultYaml);
+    IPropertyTree * allowed = defaults->queryPropTree("udpsim");
+    for (unsigned argNo = 1; argNo < argc; argNo++)
+    {
+        const char *arg = argv[argNo];
+        if (arg[0]=='-' && arg[1]=='-')
+        {
+            arg += 2;
+            StringBuffer attrname("@");
+            const char * eq = strchr(arg, '=');
+            if (eq)
+                attrname.append(eq-arg, arg);
+            else
+                attrname.append(arg);
+            if (!allowed->hasProp(attrname))
+            {
+                printf("Unrecognized option %s\n\n", attrname.str()+1);
+                usage();
+            }
+            if (!eq && !isBoolean(allowed->queryProp(attrname)))
+            {
+                printf("Option %s requires a value\n\n", attrname.str()+1);
+                usage();
+            }
+        }
+        else
+        {
+            printf("Unexpected argument %s\n\n", arg);
+            usage();
+        }
+    }
+
+    Owned<IPropertyTree> options = loadConfiguration(defaultYaml, argv, "udpsim", "UDPSIM", nullptr, nullptr);
+    if (options->getPropBool("@help", false))
+        usage();
+#ifdef TEST_DROPPED_PACKETS
+    udpDropDataPackets = options->getPropBool("@dropDataPackets", false);
+    udpDropFlowPackets[flowType::ok_to_send] = options->getPropInt("@dropOkToSendPackets", 0);  // drop 1 in N
+    udpDropFlowPackets[flowType::request_received] = options->getPropInt("@dropRequestReceivedPackets", 0);  // drop 1 in N
+    udpDropFlowPackets[flowType::request_to_send] = options->getPropInt("@dropRequestToSendPackets", 0);  // drop 1 in N
+    udpDropFlowPackets[flowType::request_to_send_more] = options->getPropInt("@dropRequestToSendMorePackets", 0);  // drop 1 in N
+    udpDropFlowPackets[flowType::send_completed] = options->getPropInt("@dropSendCompletedPackets", 0);  // drop 1 in N
+#endif
+    numThreads = options->getPropInt("@numThreads", 0);
+    udpTraceLevel = options->getPropInt("@udpTraceLevel", 1);
+    udpTraceTimeouts = options->getPropBool("@udpTraceTimeouts", true);
+    udpResendLostPackets = options->getPropBool("@udpResendLostPackets", true);
+    udpRequestToSendTimeout = options->getPropInt("@udpRequestToSendTimeout", 1000);
+    udpRequestToSendAckTimeout = options->getPropInt("@udpRequestToSendAckTimeout", 1000);
+    udpMaxPendingPermits = options->getPropInt("@udpMaxPendingPermits", 1);
+    udpTraceFlow = options->getPropBool("@udpTraceFlow", false);
+
+    isUdpTestMode = true;
+    roxiemem::setTotalMemoryLimit(false, false, false, 20*1024*1024, 0, NULL, NULL);
+    dbm.setown(roxiemem::createDataBufferManager(roxiemem::DATA_ALIGNMENT_SIZE));
+}
+
+void simulateTraffic()
+{
+    constexpr unsigned numReceiveSlots = 100;
+    constexpr unsigned maxSlotsPerClient = 100;
+    constexpr unsigned maxSendQueueSize = 100;
+    try
+    {
+        myNode.setIp(IpAddress("1.2.3.4"));
+        Owned<IReceiveManager> rm = createReceiveManager(CCD_SERVER_FLOW_PORT, CCD_DATA_PORT, CCD_CLIENT_FLOW_PORT, numReceiveSlots, maxSlotsPerClient, false);
+        unsigned begin = msTick();
+        printf("Start test\n");
+        asyncFor(numThreads, numThreads, [maxSendQueueSize](unsigned i)
+        {
+            unsigned header = 0;
+            IpAddress pretendIP(VStringBuffer("8.8.8.%d", i));
+            // Note - this is assuming we send flow on the data port (that option defaults true in roxie too)
+            Owned<ISendManager> sm = createSendManager(CCD_DATA_PORT, CCD_DATA_PORT, CCD_CLIENT_FLOW_PORT, maxSendQueueSize, 3, pretendIP, nullptr, false);
+            Owned<IMessagePacker> mp = sm->createMessagePacker(0, 0, &header, sizeof(header), myNode, 0);
+            for (unsigned j = 0; j < 10000; j++)
+            {
+                void *buf = mp->getBuffer(500, false);
+                memset(buf, i, 500);
+                mp->putBuffer(buf, 500, false);
+            }
+            mp->flush();
+
+            //wait until all the packets have been sent and acknowledged
+            while(!sm->allDone())
+                Sleep(50);
+        });
+        printf("End test %u\n", msTick() - begin);
+    }
+    catch (IException * e)
+    {
+        StringBuffer msg;
+        printf("Exception: %s\n", e->errorMessage(msg).str());
+    }
+}
+
+int main(int argc, const char **argv)
+{
+    InitModuleObjects();
+    strdup("Make sure leak checking is working");
+    queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time|MSGFIELD_microTime|MSGFIELD_milliTime|MSGFIELD_thread);
+    initOptions(argc, argv);
+    simulateTraffic();
+    ExitModuleObjects();
+    releaseAtoms();
+    return 0;
+}
+
+#else
+
+int main(int argc, const char **arv)
+{
+    printf("udpsim requires a build with SOCKET_SIMULATION enabled\n");
+    return 2;
+}
+
+#endif

roxie/udplib/udptrr.cpp

@@ -210,6 +210,15 @@ class CReceiveManager : implements IReceiveManager, public CInterface
                     StringBuffer ipStr;
                     DBGLOG("UdpReceiver: sending request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getIpText(ipStr).str());
                 }
+#ifdef TEST_DROPPED_PACKETS
+                flowPacketsSent[msg.cmd]++;
+                if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd]==0)
+                {
+                    StringBuffer ipStr;
+                    DBGLOG("UdpReceiver: deliberately dropping request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getIpText(ipStr).str());
+                }
+                else
+#endif
                 flowSocket->write(&msg, udpResendLostPackets ? sizeof(UdpPermitToSendMsg) : offsetof(UdpPermitToSendMsg, seen));
                 flowPermitsSent++;
 
@@ -241,6 +250,15 @@ class CReceiveManager : implements IReceiveManager, public CInterface
                     StringBuffer ipStr;
                     DBGLOG("UdpReceiver: sending ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getIpText(ipStr).str());
                 }
+#ifdef TEST_DROPPED_PACKETS
+                flowPacketsSent[msg.cmd]++;
+                if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd]==0)
+                {
+                    StringBuffer ipStr;
+                    DBGLOG("UdpReceiver: deliberately dropping ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getIpText(ipStr).str());
+                }
+                else
+#endif
                 flowSocket->write(&msg, udpResendLostPackets ? sizeof(UdpPermitToSendMsg) : offsetof(UdpPermitToSendMsg, seen));
                 flowPermitsSent++;
             }

roxie/udplib/udptrs.cpp

@@ -40,10 +40,6 @@ unsigned udpMaxRetryTimedoutReqs = 0; // 0 means off (keep retrying forever)
 unsigned udpRequestToSendTimeout = 0; // value in milliseconds - 0 means calculate from query timeouts
 unsigned udpRequestToSendAckTimeout = 10; // value in milliseconds
 
-#ifdef _DEBUG
-//#define TEST_DROPPED_PACKETS
-#endif
-
 using roxiemem::DataBuffer;
 /*
  *
@@ -216,6 +212,15 @@ private:
                 StringBuffer s, s2;
                 DBGLOG("UdpSender[%s]: sending flowType::%s msg %" SEQF "u flowSeq %" SEQF "u to node=%s", msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, ip.getIpText(s).str());
             }
+#ifdef TEST_DROPPED_PACKETS
+            flowPacketsSent[msg.cmd]++;
+            if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd] == 0)
+            {
+                StringBuffer s, s2;
+                DBGLOG("UdpSender[%s]: deliberately dropping flowType::%s msg %" SEQF "u flowSeq %" SEQF "u to node=%s", msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, ip.getIpText(s).str());
+            }
+            else
+#endif
             send_flow_socket->write(&msg, sizeof(UdpRequestToSendMsg));
             flowRequestsSent++;
         }
@@ -418,7 +423,7 @@ public:
             try
             {
 #ifdef TEST_DROPPED_PACKETS
-                if (((header->pktSeq & UDP_PACKET_RESENT)==0) && (header->pktSeq==0 || header->pktSeq==10 || ((header->pktSeq&UDP_PACKET_COMPLETE) != 0)))
+                if (udpDropDataPackets && ((header->pktSeq & UDP_PACKET_RESENT)==0) && (header->pktSeq==0 || header->pktSeq==10 || ((header->pktSeq&UDP_PACKET_COMPLETE) != 0)))
                     DBGLOG("Deliberately dropping packet %" SEQF "u", header->sendSeq);
                 else
 #endif
@@ -787,8 +792,8 @@ class CSendManager : implements ISendManager, public CInterface
         ~send_receive_flow() 
         {
             running = false;
-            if (flow_socket) 
-                flow_socket->close();
+            if (flow_socket)
+                flow_socket->shutdownNoThrow();
             join();
         }
         

system/jlib/jptree.cpp

@@ -8414,7 +8414,7 @@ void mergeConfiguration(IPropertyTree & target, const IPropertyTree & source, co
         bool first = false;
         bool endprior = false;
         bool sequence = checkInSequence(child, seqname, first, endprior);
-        if (first && (!name || isScalarItem(child))) //arrays of unamed objects or scalars are replaced
+        if (first && (!name || isScalarItem(child))) //arrays of unnamed objects or scalars are replaced
             target.removeProp(tag);
 
         IPropertyTree * match = ensureMergeConfigTarget(target, tag, altname ? altNameAttribute : "@name", name, sequence);
@@ -8565,7 +8565,7 @@ static void applyCommandLineOption(IPropertyTree * config, const char * option,
     config->setProp(path, value);
 }
 
-static void applyCommandLineOption(IPropertyTree * config, const char * option, std::initializer_list<const char *> ignoreOptions)
+static void applyCommandLineOption(IPropertyTree * config, const char * option, std::initializer_list<const std::string> ignoreOptions)
 {
     const char * eq = strchr(option, '=');
     StringBuffer name;
@@ -8638,7 +8638,7 @@ Owned<IPropertyTree> getGlobalConfigSP()
     return getGlobalConfig();
 }
 
-jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const char *> ignoreOptions)
+jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const std::string> ignoreOptions)
 {
     for (const char * * pArg = argv; *pArg; pArg++)
     {

system/jlib/jptree.hpp

@@ -316,7 +316,7 @@ inline static bool isValidXPathChr(char c)
 //export for unit test
 jlib_decl void mergeConfiguration(IPropertyTree & target, const IPropertyTree & source, const char *altNameAttribute=nullptr, bool overwriteAttr=true);
 
-jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const char *> ignoreOptions = {});
+jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv, std::initializer_list<const std::string> ignoreOptions = {});
 jlib_decl IPropertyTree * loadConfiguration(IPropertyTree * defaultConfig, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute=nullptr, bool monitor=true);
 jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *), const char *altNameAttribute=nullptr, bool monitor=true);
 jlib_decl IPropertyTree * getCostsConfiguration();

system/jlib/jsocket.cpp

@@ -420,6 +420,7 @@ public:
     void        errclose();
     bool        connectionless() { return (sockmode!=sm_tcp)&&(sockmode!=sm_tcp_server); }
     void        shutdown(unsigned mode=SHUTDOWN_READWRITE);
+    void        shutdownNoThrow(unsigned mode);
 
     ISocket*    accept(bool allowcancel, SocketEndpoint *peerEp=nullptr);
     int         wait_read(unsigned timeout);
@@ -2521,6 +2522,17 @@ void CSocket::shutdown(unsigned mode)
     }
 }
 
+void CSocket::shutdownNoThrow(unsigned mode)
+{
+    if (state == ss_open) {
+        state = ss_shutdown;
+#ifdef SOCKTRACE
+        PROGLOG("SOCKTRACE: shutdown(%d) socket %x %d (%p)", mode, sock, sock, this);
+#endif
+        ::shutdown(sock, mode);
+    }
+}
+
 void CSocket::errclose()
 {
 #ifdef USERECVSEM

system/jlib/jsocket.hpp

@@ -340,6 +340,9 @@ public:
     //
     virtual void  shutdown(unsigned mode=SHUTDOWN_READWRITE) = 0; // not needed for UDP
 
+    // Same as shutdown, but never throws an exception (to call from closedown destructors)
+    virtual void  shutdownNoThrow(unsigned mode=SHUTDOWN_READWRITE) = 0; // not needed for UDP
+
     // Get local name of accepted (or connected) socket and returns port
     virtual int name(char *name,size32_t namemax)=0;
 

system/security/securesocket/securesocket.cpp

@@ -245,6 +245,12 @@ public:
         m_socket->shutdown(mode);
     }
 
+    // Same as shutdown, but never throws an exception (to call from closedown destructors)
+    virtual void  shutdownNoThrow(unsigned mode)
+    {
+        m_socket->shutdownNoThrow(mode);
+    }
+
     // Get local name of accepted (or connected) socket and returns port
     virtual int name(char *name,size32_t namemax)
     {