Merge pull request #5487 from GordonSmith/HPCC-10724

HPCC-10724 Implement disk space usage for a scope/owner

Reviewed-By: Miguel Vazquez <miguel.vazquez@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

esp/files/scripts/DiskUsageWidget.js

@@ -0,0 +1,111 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/on",
+
+    "dgrid/OnDemandGrid",
+    "dgrid/Keyboard",
+    "dgrid/Selection",
+    "dgrid/selector",
+    "dgrid/extensions/ColumnResizer",
+    "dgrid/extensions/DijitRegistry",
+
+    "hpcc/WsDfu",
+    "hpcc/ESPUtil",
+    "hpcc/FilterDropDownWidget",
+
+    "dojo/text!../templates/DiskUsageWidget.html"
+], function (declare, lang, i18n, nlsHPCC, on,
+                OnDemandGrid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry,
+                WsDfu, ESPUtil, FilterDropDownWidget,
+                template) {
+    return declare("DiskUsageWidget", [_Widget, ESPUtil.FormHelper], {
+        templateString: template,
+        baseClass: "DiskUsageWidget",
+        i18n: nlsHPCC,
+
+        resize: function (args) {
+            this.inherited(arguments);
+            this.widget.BorderContainer.resize();
+        },
+
+        getTitle: function () {
+            return this.i18n.title_DiskUsage;
+        },
+
+        //  Hitched actions  ---
+        _onRefresh: function (event) {
+            this.refreshGrid();
+        },
+
+        //  Implementation  ---
+        init: function (params) {
+            if (this.inherited(arguments))
+                return;
+
+            this.initDiskUsageGrid();
+
+            this.widget.Filter.refreshState();
+            var context = this;
+            this.widget.Filter.on("clear", function (evt) {
+                context.refreshGrid();
+            });
+            this.widget.Filter.on("apply", function (evt) {
+                context.refreshGrid();
+            });
+        },
+
+        initDiskUsageGrid: function () {
+            var store = new WsDfu.CreateDiskUsageStore();
+            this.diskUsageGrid = new declare([OnDemandGrid, Keyboard, Selection, ColumnResizer, DijitRegistry, ESPUtil.GridHelper])({
+                allowSelectAll: true,
+                deselectOnRefresh: false,
+                store: store,
+                query: this.getFilter(),
+                columns: {
+                    Name: { label: this.i18n.Grouping, width: 90, sortable: true },
+                    NumOfFiles: { label: this.i18n.FileCounts, width: 90, sortable: true },
+                    TotalSize: { label: this.i18n.TotalSize, width: 90, sortable: true },
+                    LargestFile: { label: this.i18n.LargestFile, sortable: true },
+                    LargestSize: { label: this.i18n.LargestSize, width: 90, sortable: true },
+                    SmallestFile: { label: this.i18n.SmallestFile, sortable: true },
+                    SmallestSize: { label: this.i18n.SmallestSize, width: 90, sortable: true },
+                    NumOfFilesUnknown: { label: this.i18n.FilesWithUnknownSize, width: 90, sortable: true }
+                }
+            }, this.id + "DiskUsageGrid");
+        },
+
+        getFilter: function () {
+            var retVal = this.widget.Filter.toObject();
+            lang.mixin(retVal, {
+                StartDate: this.getISOString("FromDate", "FromTime"),
+                EndDate: this.getISOString("ToDate", "ToTime")
+            });
+            return retVal;
+        },
+
+        refreshGrid: function (clearSelection) {
+            this.diskUsageGrid.set("query", this.getFilter());
+            if (clearSelection) {
+                this.diskUsageGrid.clearSelection();
+            }
+        }
+    });
+});

esp/files/scripts/HPCCPlatformOpsWidget.js

@@ -22,6 +22,7 @@ define([
     "dijit/registry",
 
     "hpcc/_TabContainerWidget",
+    "hpcc/DiskUsageWidget",
     "hpcc/ESPRequest",
 
     "dojo/text!../templates/HPCCPlatformOpsWidget.html",
@@ -33,7 +34,7 @@ define([
 
 ], function (declare, lang, i18n, nlsHPCC,
                 registry,
-                _TabContainerWidget, ESPRequest,
+                _TabContainerWidget, DiskUsageWidget, ESPRequest,
                 template) {
     return declare("HPCCPlatformOpsWidget", [_TabContainerWidget], {
         templateString: template,

esp/files/scripts/HPCCPlatformWidget.js

@@ -31,6 +31,7 @@ define([
     "hpcc/ESPActivity",
     "hpcc/ws_account",
     "hpcc/ws_access",
+    "hpcc/WsDfu",
     "hpcc/WsSMC",
     "hpcc/GraphWidget",
 
@@ -58,7 +59,7 @@ define([
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domStyle,
                 registry, Tooltip,
                 UpgradeBar,
-                _TabContainerWidget, ESPActivity, WsAccount, WsAccess, WsSMC, GraphWidget,
+                _TabContainerWidget, ESPActivity, WsAccount, WsAccess, WsDfu, WsSMC, GraphWidget,
                 template) {
     return declare("HPCCPlatformWidget", [_TabContainerWidget], {
         templateString: template,
@@ -122,17 +123,50 @@ define([
             }
         },
 
+        refreshUserName: function () {
+            var userDisplay = this.userName ? this.userName : "";
+
+            var total = 0;
+            var myTotal = 0;
+            arrayUtil.forEach(this.spaceUsage, function (item, idx) {
+                var itemTotal = item.TotalSize.split(",").join("");
+                total += parseInt(itemTotal);
+                if (item.Name === this.userName) {
+                    myTotal = parseInt(itemTotal);
+                }
+            }, this);
+            this.userUsage = (myTotal / total) * 100;
+
+            if (this.userUsage) {
+                userDisplay += " (" + (Math.round(this.userUsage * 100) / 100) + "%)"
+            }
+            dom.byId(this.id + "UserID").innerHTML = userDisplay;
+        },
+
         init: function (params) {
             if (this.inherited(arguments))
                 return;
-            var context = this;
-            registry.byId(context.id + "SetBanner").set("disabled", true);
 
+            registry.byId(this.id + "SetBanner").set("disabled", true);
+
+            var context = this;
             WsAccount.MyAccount({
             }).then(function (response) {
                 if (lang.exists("MyAccountResponse.username", response)) {
-                    dom.byId(context.id + "UserID").innerHTML = response.MyAccountResponse.username;
-                    context.checkIfAdmin(response.MyAccountResponse.username);
+                    context.userName = response.MyAccountResponse.username;
+                    context.checkIfAdmin(context.username);
+                    context.refreshUserName();
+                }
+            });
+
+            WsDfu.DFUSpace({
+                request: {
+                    CountBy: "Owner"
+                }
+            }).then(function (response) {
+                if (lang.exists("DFUSpaceResponse.DFUSpaceItems.DFUSpaceItem", response)) {
+                    context.spaceUsage = response.DFUSpaceResponse.DFUSpaceItems.DFUSpaceItem;
+                    context.refreshUserName();
                 }
             });
 

esp/files/scripts/WsDfu.js

@@ -19,17 +19,80 @@ define([
     "dojo/_base/lang",
     "dojo/_base/Deferred",
     "dojo/_base/array",
+    "dojo/store/Memory",
+    "dojo/store/Observable",
     "dojo/store/util/QueryResults",
 
     "dojox/xml/parser",
 
     "hpcc/ESPBase",
     "hpcc/ESPRequest"
-], function (declare, lang, Deferred, arrayUtil, QueryResults,
+], function (declare, lang, Deferred, arrayUtil, Memory, Observable, QueryResults,
     parser,
     ESPBase, ESPRequest) {
 
-    return {
+    var DiskUsageStore = declare([Memory], {
+
+        constructor: function () {
+            this.idProperty = "__hpcc_id";
+        },
+
+        query: function (query, options) {
+            switch (query.CountBy) {
+                case "Year":
+                case "Quarter":
+                case "Month":
+                case "Day":
+                    query.Interval = query.CountBy;
+                    query.CountBy = "Date";
+                    break;
+            }
+            var results = self.DFUSpace({
+                request: query
+            }).then(lang.hitch(this, function (response) {
+                var data = [];
+                if (lang.exists("DFUSpaceResponse.DFUSpaceItems.DFUSpaceItem", response)) {
+                    arrayUtil.forEach(response.DFUSpaceResponse.DFUSpaceItems.DFUSpaceItem, function (item, idx) {
+                        data.push(lang.mixin(item, {
+                            __hpcc_id: item.Name
+                        }));
+                    }, this);
+                }
+                if (options.sort && options.sort.length) {
+                    data.sort(function (_l, _r) {
+                        var l = _l[options.sort[0].attribute];
+                        var r = _r[options.sort[0].attribute];
+                        if (l === r) {
+                            return 0;
+                        }
+                        switch (options.sort[0].attribute) {
+                            case "TotalSize":
+                            case "LargestSize":
+                            case "SmallestSize":
+                            case "NumOfFiles":
+                            case "NumOfFilesUnknown":
+                                l = parseInt(l.split(",").join(""));
+                                r = parseInt(r.split(",").join(""));
+                        }
+                        if (options.sort[0].descending) {
+                            return r < l ? -1 : 1;
+                        }
+                        return l < r ? -1 : 1;
+                    })
+                }
+                this.setData(data);
+                return this.data;
+            }));
+            return QueryResults(results);
+        }
+    });
+
+    var self = {
+        CreateDiskUsageStore: function() {
+            var store = new DiskUsageStore();
+            return Observable(store);
+        },
+
         DFUArrayAction: function (logicalFiles, actionType, callback) {
             arrayUtil.forEach(logicalFiles, function (item, idx) {
                 item.qualifiedName = item.Name + "@" + item.ClusterName;
@@ -130,6 +193,10 @@ define([
             return ESPRequest.send("WsDfu", "DFUQuery", params);
         },
 
+        DFUSpace: function (params) {
+            return ESPRequest.send("WsDfu", "DFUSpace", params);
+        },
+
         DFUInfo: function (params) {
             return ESPRequest.send("WsDfu", "DFUInfo", params);
         },
@@ -152,5 +219,7 @@ define([
             });
         }
     };
+
+    return self;
 });
 

esp/files/scripts/nls/hpcc.js

@@ -54,6 +54,7 @@ define({root:
     Dali: "Dali",
     dataset: ":=dataset*",
     Date: "Date",
+    Day: "Day",
     Deactivate: "Deactivate",
     DEF: "DEF",
     Delete: "Delete",
@@ -76,6 +77,7 @@ define({root:
     Despray: "Despray",
     DFUServerName: "DFU Server Name",
     Directory: "Directory",
+    DiskUsage: "Disk Usage",
     Distance: "Distance",
     Dll: "Dll",
     DOT: "DOT",
@@ -109,12 +111,14 @@ define({root:
     FetchingData: "Fetching Data...",
     fetchingresults: "fetching results",
     File: "File",
+    FileCounts: "File Counts",
     FileName: "File Name",
     FileParts: "File Parts",
     FilePath: "File Path",
     Files: "Files",
     FileSize: "File Size",
     FilesNoPackage: "Files without matching package definitions",
+    FilesWithUnknownSize: "Files With Unknown Size",
     FileType: "File Type",
     FileUploader: "File Uploader",
     Filter: "Filter",
@@ -130,6 +134,8 @@ define({root:
     Graphs: "Graphs",
     GraphView: "Graph View",
     Group: "Group",
+    GroupBy: "Group By",
+    Grouping: "Grouping",
     GroupDetails: "Group Details",
     GroupName: "Group Name",
     GroupPermissions: "Group Permissions",
@@ -158,6 +164,8 @@ define({root:
     LandingZone: "Landing Zone",
     LandingZones: "Landing Zones",
     Largest: "Largest",
+    LargestFile: "Largest File",
+    LargestSize: "Largest Size",
     LastName: "Last Name",
     LastNDays: "Last N Days",
     LegacyForm: "Legacy Form",
@@ -192,6 +200,7 @@ define({root:
     MonitorEventName: "Monitor Event Name",
     MonitorShotLimit: "Monitor Shot Limit",
     MonitorSub: "Monitor Sub",
+    Month: "Month",
     MustContainUppercaseAndSymbol: "Must contain uppercase and symbol",
     NA: "N/A",
     Name: "Name",
@@ -259,6 +268,7 @@ define({root:
     Publish: "Publish",
     PublishedBy: "Published By",
     PublishedQueries: "Published Queries",
+    Quarter: "Quarter",
     Queries: "Queries",
     QueriesNoPackage: "Queries without matching package",
     QueryDetailsfor: "Query Details for",
@@ -311,6 +321,8 @@ define({root:
     Skew: "Skew",
     Slaves: "Slaves",
     Smallest: "Smallest",
+    SmallestFile: "Smallest File",
+    SmallestSize: "Smallest Size",
     SOAP: "SOAP",
     SomeDescription: "Some*Description",
     somefile: "*::somefile*",
@@ -354,6 +366,7 @@ define({root:
     title_Activity: "Activity",
     title_DFUQuery: "Logical Files",
     title_DFUWUDetails: "DFU Workunit",
+    title_DiskUsage: "Disk Usage",
     title_ECLPlayground: "ECL Playground",
     title_EventScheduleWorkunit: "Event Scheduler",
     title_GetDFUWorkunits: "DFU Workunits",
@@ -388,6 +401,7 @@ define({root:
     ToDate: "To Date",
     Toenablegraphviews: "To enable graph views, please install the Graph View Control plugin",
     ToSizes: "To Sizes",
+    TotalSize: "Total Size",
     TotalThorTime: "Total Thor Time",
     Type: "Type",
     undefined: "undefined",
@@ -433,6 +447,7 @@ define({root:
     XLS: "XLS",
     XML: "XML",
     XRef: "XRef",
+    Year: "Year",
     ZAP: "Z.A.P",
     ZeroLogicalFilesCheckFilter: "Zero Logical Files(check filter)",
     Zip: "Zip",

esp/files/templates/DiskUsageWidget.html

@@ -0,0 +1,30 @@
+<div class="${baseClass}">
+    <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%;" data-dojo-type="dijit.layout.BorderContainer">
+        <div id="${id}Toolbar" class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
+            <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+            <span data-dojo-type="dijit.ToolbarSeparator"></span>
+            <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
+                <select id="${id}GroupBy" title="${i18n.GroupBy}:" name="CountBy" colspan="2" data-dojo-type="dijit/form/Select">
+                    <option selected="selected" value="Owner">${i18n.Owner}</option>
+                    <option value="Scope">${i18n.Scope}</option>
+                    <option value="Year">${i18n.Year}</option>
+                    <option value="Quarter">${i18n.Quarter}</option>
+                    <option value="Month">${i18n.Month}</option>
+                    <option value="Day">${i18n.Day}</option>
+                </select>
+                <input id="${id}ScopeUnder" title="${i18n.Scope}:" name="ScopeUnder" colspan="2" style="width:100%" data-dojo-type="dijit.form.TextBox" />
+                <input id="${id}OwnerUnder" title="${i18n.Owner}:" name="OwnerUnder" colspan="2" style="width:100%" data-dojo-type="dijit.form.TextBox" />
+                <input id="${id}FromDate" title="${i18n.FromDate}:" name="StartDate" data-dojo-props="trim: true, placeHolder:'7/28/2013'" data-dojo-type="dijit.form.DateTextBox" />
+                <input id="${id}FromTime" name="FromTime" data-dojo-props="trim: true, placeHolder:'7:30 AM'" data-dojo-type="dijit.form.TimeTextBox" />
+                <input id="${id}ToDate" title="${i18n.ToDate}:" name="EndDate" data-dojo-props="trim: true, placeHolder:'7/28/2013'" data-dojo-type="dijit.form.DateTextBox" />
+                <input id="${id}ToTime" name="ToTime" data-dojo-props="trim: true, placeHolder:'7:30 PM'" data-dojo-type="dijit.form.TimeTextBox" />
+            </div>
+            <span data-dojo-type="dijit.ToolbarSeparator"></span>
+            <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
+        </div>
+        <div id="${id}DiskUsageGridCP" style="border:0px; padding: 0px; border-color:none" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+            <div id="${id}DiskUsageGrid">
+            </div>
+        </div>
+    </div>
+</div>

esp/files/templates/HPCCPlatformOpsWidget.html

@@ -4,6 +4,8 @@
             <div id="${id}StackController" style="width: 100%" data-dojo-props="containerId:'${id}TabContainer'" data-dojo-type="dijit.layout.StackController"></div>
         </div>
         <div id="${id}TabContainer" data-dojo-props="region: 'center', tabPosition: 'top'" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.StackContainer">
+            <div id="${id}_DiskUsage" title="${i18n.DiskUsage}" style="padding: 0px; border:0px; border-color:none; overflow: hidden" data-dojo-type="DiskUsageWidget">
+            </div>
             <div id="${id}_TargetClusters" title="${i18n.TargetClusters}" style="padding: 0px; border:0px; border-color:none; overflow: hidden" data-dojo-type="dijit.layout.ContentPane">
             </div>
             <div id="${id}_ClusterProcesses" title="${i18n.ClusterProcesses}" style="padding: 0px; border:0px; border-color:none; overflow: hidden" data-dojo-type="dijit.layout.ContentPane">