jstats.cpp 111 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jiface.hpp"
  14. #include "jstats.h"
  15. #include "jexcept.hpp"
  16. #include "jiter.ipp"
  17. #include "jlog.hpp"
  18. #include "jregexp.hpp"
  19. #include "jfile.hpp"
  20. #include "jerror.hpp"
  21. #include <math.h>
  22. #include <sstream>
  23. #include <iomanip>
  24. #include <ctype.h>
  25. #include <cmath>
  26. #ifdef _WIN32
  27. #include <sys/timeb.h>
  28. #endif
  29. static CriticalSection statsNameCs;
  30. static StringBuffer statisticsComponentName;
  31. static StatisticCreatorType statisticsComponentType = SCTunknown;
  32. const static unsigned currentStatisticsVersion = 1;
  33. StatisticCreatorType queryStatisticsComponentType()
  34. {
  35. return statisticsComponentType;
  36. }
  37. const char * queryStatisticsComponentName()
  38. {
  39. CriticalBlock c(statsNameCs);
  40. if (statisticsComponentName.length() == 0)
  41. {
  42. statisticsComponentName.append("unknown").append(GetCachedHostName());
  43. DBGLOG("getProcessUniqueName hasn't been configured correctly");
  44. }
  45. return statisticsComponentName.str();
  46. }
  47. void setStatisticsComponentName(StatisticCreatorType processType, const char * processName, bool appendIP)
  48. {
  49. if (!processName)
  50. return;
  51. CriticalBlock c(statsNameCs);
  52. statisticsComponentType = processType;
  53. statisticsComponentName.clear().append(processName);
  54. if (appendIP)
  55. statisticsComponentName.append("@").append(GetCachedHostName()); // should I use _ instead?
  56. }
  57. //--------------------------------------------------------------------------------------------------------------------
  58. // Textual forms of the different enumerations, first items are for none and all.
  59. static constexpr const char * const measureNames[] = { "", "all", "ns", "ts", "cnt", "sz", "cpu", "skw", "node", "ppm", "ip", "cy", "en", "txt", "bool", "id", "fname", "cost", NULL };
  60. static constexpr const char * const creatorTypeNames[]= { "", "all", "unknown", "hthor", "roxie", "roxie:s", "thor", "thor:m", "thor:s", "eclcc", "esp", "summary", NULL };
  61. static constexpr const char * const scopeTypeNames[] = { "", "all", "global", "graph", "subgraph", "activity", "allocator", "section", "compile", "dfu", "edge", "function", "workflow", "child", "file", "channel", "unknown", nullptr };
  62. static unsigned matchString(const char * const * names, const char * search, unsigned dft)
  63. {
  64. if (!search)
  65. return dft;
  66. if (streq(search, "*"))
  67. search = "all";
  68. unsigned i=0;
  69. for (;;)
  70. {
  71. const char * next = names[i];
  72. if (!next)
  73. return dft;
  74. if (strieq(next, search))
  75. return i;
  76. i++;
  77. }
  78. }
  79. //--------------------------------------------------------------------------------------------------------------------
  80. static const StatisticScopeType scoreOrder[] = {
  81. SSTedge,
  82. SSTactivity,
  83. SSTnone,
  84. SSTall,
  85. SSTglobal,
  86. SSTgraph,
  87. SSTsubgraph,
  88. SSTallocator,
  89. SSTsection,
  90. SSTcompilestage,
  91. SSTdfuworkunit,
  92. SSTfunction,
  93. SSTworkflow,
  94. SSTchildgraph,
  95. SSTfile,
  96. SSTchannel, // MORE - not sure what this means!
  97. SSTunknown
  98. };
  99. static int scopePriority[SSTmax];
  100. MODULE_INIT(INIT_PRIORITY_STANDARD)
  101. {
  102. static_assert(_elements_in(scoreOrder) == SSTmax, "Elements missing from scoreOrder[]");
  103. for (unsigned i=0; i < _elements_in(scoreOrder); i++)
  104. scopePriority[scoreOrder[i]] = i;
  105. return true;
  106. }
  107. extern jlib_decl int compareScopeName(const char * left, const char * right)
  108. {
  109. if (!left || !*left)
  110. {
  111. if (!right || !*right)
  112. return 0;
  113. else
  114. return -1;
  115. }
  116. else
  117. {
  118. if (!right || !*right)
  119. return +1;
  120. }
  121. StatsScopeId leftId;
  122. StatsScopeId rightId;
  123. for(;;)
  124. {
  125. leftId.extractScopeText(left, &left);
  126. rightId.extractScopeText(right, &right);
  127. int result = leftId.compare(rightId);
  128. if (result != 0)
  129. return result;
  130. left = strchr(left, ':');
  131. right = strchr(right, ':');
  132. if (!left || !right)
  133. {
  134. if (left)
  135. return +1;
  136. if (right)
  137. return -1;
  138. return 0;
  139. }
  140. left++;
  141. right++;
  142. }
  143. }
  144. //--------------------------------------------------------------------------------------------------------------------
  145. extern jlib_decl unsigned __int64 getTimeStampNowValue()
  146. {
  147. #ifdef _WIN32
  148. struct _timeb now;
  149. _ftime(&now);
  150. return (unsigned __int64)now.time * I64C(1000000) + now.millitm * 1000;
  151. #else
  152. struct timeval tm;
  153. gettimeofday(&tm,NULL);
  154. return (unsigned __int64)tm.tv_sec * I64C(1000000) + tm.tv_usec;
  155. #endif
  156. }
  157. unsigned __int64 getIPV4StatsValue(const IpAddress & ip)
  158. {
  159. unsigned ipValue;
  160. if (ip.getNetAddress(sizeof(ipValue),&ipValue))
  161. return ipValue;
  162. return 0;
  163. }
  164. //--------------------------------------------------------------------------------------------------------------------
  165. const static unsigned __int64 oneMicroSecond = I64C(1000);
  166. const static unsigned __int64 oneMilliSecond = I64C(1000000);
  167. const static unsigned __int64 oneSecond = I64C(1000000000);
  168. const static unsigned __int64 oneMinute = I64C(60000000000);
  169. const static unsigned __int64 oneHour = I64C(3600000000000);
  170. const static unsigned __int64 oneDay = 24 * I64C(3600000000000);
  171. static void formatTime(StringBuffer & out, unsigned __int64 value)
  172. {
  173. //Aim to display at least 3 significant digits in the result string
  174. if (value < oneMicroSecond)
  175. out.appendf("%uns", (unsigned)value);
  176. else if (value < oneMilliSecond)
  177. {
  178. unsigned uvalue = (unsigned)value;
  179. out.appendf("%u.%03uus", uvalue / 1000, uvalue % 1000);
  180. }
  181. else if (value < oneSecond)
  182. {
  183. unsigned uvalue = (unsigned)value;
  184. out.appendf("%u.%03ums", uvalue / 1000000, (uvalue / 1000) % 1000);
  185. }
  186. else
  187. {
  188. unsigned days = (unsigned)(value / oneDay);
  189. value = value % oneDay;
  190. unsigned hours = (unsigned)(value / oneHour);
  191. value = value % oneHour;
  192. unsigned mins = (unsigned)(value / oneMinute);
  193. value = value % oneMinute;
  194. unsigned secs = (unsigned)(value / oneSecond);
  195. unsigned ns = (unsigned)(value % oneSecond);
  196. if (days > 0)
  197. out.appendf("%u days ", days);
  198. if (hours > 0 || days)
  199. out.appendf("%uh%02um%02us", hours, mins, secs);
  200. else if (mins >= 10)
  201. out.appendf("%um%02us", mins, secs);
  202. else if (mins >= 1)
  203. out.appendf("%um%02u.%03us", mins, secs, ns / 1000000);
  204. else
  205. out.appendf("%u.%03us", secs, ns / 1000000);
  206. }
  207. }
  208. extern void formatTimeCollatable(StringBuffer & out, unsigned __int64 value, bool nano)
  209. {
  210. unsigned days = (unsigned)(value / oneDay);
  211. value = value % oneDay;
  212. unsigned hours = (unsigned)(value / oneHour);
  213. value = value % oneHour;
  214. unsigned mins = (unsigned)(value / oneMinute);
  215. value = value % oneMinute;
  216. unsigned secs = (unsigned)(value / oneSecond);
  217. unsigned ns = (unsigned)(value % oneSecond);
  218. if (days)
  219. out.appendf(" %3ud ", days); // Two leading spaces helps the cassandra driver force to a single partition
  220. else
  221. out.appendf(" ");
  222. if (nano)
  223. out.appendf("%2u:%02u:%02u.%09u", hours, mins, secs, ns);
  224. else
  225. out.appendf("%2u:%02u:%02u.%03u", hours, mins, secs, ns/1000000);
  226. // More than 999 days, I don't care that it goes wrong.
  227. }
  228. extern unsigned __int64 extractTimestamp(const char *s, const char * * end)
  229. {
  230. if (!s)
  231. return 0;
  232. const char * next = nullptr;
  233. CDateTime timestamp;
  234. try
  235. {
  236. timestamp.setString(s, &next, false); // Sets to date and time given as yyyy-mm-ddThh:mm:ss[.nnnnnnnnn]
  237. }
  238. catch(IException * e)
  239. {
  240. e->Release();
  241. return 0;
  242. }
  243. //Skip a trailing Z. The setString function should really process it, but that may break other code in CScmDateTime.
  244. if (*next == 'Z')
  245. next++;
  246. if (end)
  247. *end = next;
  248. return timestamp.getTimeStamp();
  249. }
  250. extern unsigned __int64 extractTimeCollatable(const char *s, const char * * end)
  251. {
  252. if (!s)
  253. return 0;
  254. unsigned days,hours,mins;
  255. double secs;
  256. unsigned numRead = 0;
  257. if (sscanf(s, " %ud %u:%u:%lf%n", &days, &hours, &mins, &secs, &numRead)!=4)
  258. {
  259. days = 0;
  260. if (sscanf(s, " %u:%u:%lf%n", &hours, &mins, &secs, &numRead) != 3)
  261. return 0;
  262. }
  263. if (end)
  264. *end = s + numRead;
  265. return days*oneDay + hours*oneHour + mins*oneMinute + secs*oneSecond;
  266. }
  267. static void formatTimeStamp(StringBuffer & out, unsigned __int64 value)
  268. {
  269. time_t seconds = value / 1000000;
  270. unsigned us = value % 1000000;
  271. char timeStamp[64];
  272. time_t tNow = seconds;
  273. #ifdef _WIN32
  274. struct tm *gmtNow;
  275. gmtNow = gmtime(&tNow);
  276. strftime(timeStamp, 64, "%Y-%m-%dT%H:%M:%S", gmtNow);
  277. #else
  278. struct tm gmtNow;
  279. gmtime_r(&tNow, &gmtNow);
  280. strftime(timeStamp, 64, "%Y-%m-%dT%H:%M:%S", &gmtNow);
  281. #endif //_WIN32
  282. out.append(timeStamp).appendf(".%03uZ", us / 1000);
  283. }
  284. void formatTimeStampAsLocalTime(StringBuffer & out, unsigned __int64 value)
  285. {
  286. time_t seconds = value / 1000000;
  287. unsigned us = value % 1000000;
  288. char timeStamp[64];
  289. time_t tNow = seconds;
  290. #ifdef _WIN32
  291. struct tm *gmtNow;
  292. gmtNow = localtime(&tNow);
  293. strftime(timeStamp, 64, "%H:%M:%S", gmtNow);
  294. #else
  295. struct tm gmtNow;
  296. localtime_r(&tNow, &gmtNow);
  297. strftime(timeStamp, 64, "%H:%M:%S", &gmtNow);
  298. #endif //_WIN32
  299. out.append(timeStamp).appendf(".%03u", us / 1000);
  300. }
  301. static const unsigned oneKb = 1024;
  302. static const unsigned oneMb = 1024 * 1024;
  303. static const unsigned oneGb = 1024 * 1024 * 1024;
  304. static unsigned toPermille(unsigned x) { return (x * 1000) / 1024; }
  305. static StringBuffer & formatSize(StringBuffer & out, unsigned __int64 value)
  306. {
  307. unsigned Gb = (unsigned)(value / oneGb);
  308. unsigned Mb = (unsigned)((value % oneGb) / oneMb);
  309. unsigned Kb = (unsigned)((value % oneMb) / oneKb);
  310. unsigned b = (unsigned)(value % oneKb);
  311. if (Gb)
  312. return out.appendf("%u.%03uGb", Gb, toPermille(Mb));
  313. else if (Mb)
  314. return out.appendf("%u.%03uMb", Mb, toPermille(Kb));
  315. else if (Kb)
  316. return out.appendf("%u.%03uKb", Kb, toPermille(b));
  317. else
  318. return out.appendf("%ub", b);
  319. }
  320. static StringBuffer & formatLoad(StringBuffer & out, unsigned __int64 value)
  321. {
  322. //Stored as millionth of a core. Display as a percentage => scale by 10,000
  323. return out.appendf("%u.%03u%%", (unsigned)(value / 10000), (unsigned)(value % 10000) / 10);
  324. }
  325. static StringBuffer & formatSkew(StringBuffer & out, unsigned __int64 value)
  326. {
  327. //Skew stored as 10000 = perfect, display as percentage
  328. const __int64 sval = (__int64) value;
  329. const double percent = ((double)sval) / 100.0;
  330. if (sval >= 10000 || sval <= -10000) // For values >= 100%, whole numbers
  331. return out.appendf("%.0f%%", percent);
  332. else if (sval >= 1000 || sval <= -1000) // For values >= 10%, 1 decimal point
  333. return out.appendf("%.1f%%", percent);
  334. else // Anything < 10%, 2 decimal points
  335. return out.appendf("%.2f%%", percent);
  336. }
  337. static StringBuffer & formatIPV4(StringBuffer & out, unsigned __int64 value)
  338. {
  339. byte ip1 = (value & 255);
  340. byte ip2 = ((value >> 8) & 255);
  341. byte ip3 = ((value >> 16) & 255);
  342. byte ip4 = ((value >> 24) & 255);
  343. return out.appendf("%d.%d.%d.%d", ip1, ip2, ip3, ip4);
  344. }
  345. StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value)
  346. {
  347. return out.appendf("%.6f", cost_type2money(value));
  348. }
  349. StringBuffer & formatStatistic(StringBuffer & out, unsigned __int64 value, StatisticMeasure measure)
  350. {
  351. switch (measure)
  352. {
  353. case SMeasureNone: // Unknown stat - e.g, on old esp accessing a new workunit
  354. return out.append(value);
  355. case SMeasureTimeNs:
  356. formatTime(out, value);
  357. return out;
  358. case SMeasureTimestampUs:
  359. formatTimeStamp(out, value);
  360. return out;
  361. case SMeasureCount:
  362. return out.append(value);
  363. case SMeasureSize:
  364. return formatSize(out, value);
  365. case SMeasureLoad:
  366. return formatLoad(out, value);
  367. case SMeasureSkew:
  368. return formatSkew(out, value);
  369. case SMeasureNode:
  370. return out.append(value);
  371. case SMeasurePercent:
  372. return out.appendf("%.2f%%", (double)value / 10000.0); // stored as ppm
  373. case SMeasureIPV4:
  374. return formatIPV4(out, value);
  375. case SMeasureCycle:
  376. return out.append(value);
  377. case SMeasureBool:
  378. return out.append(boolToStr(value != 0));
  379. case SMeasureText:
  380. case SMeasureId:
  381. case SMeasureFilename:
  382. return out.append(value);
  383. case SMeasureEnum:
  384. return out.append("Enum{").append(value).append("}"); // JCS->GH for now, should map to known enum text somehow
  385. case SMeasureCost:
  386. return formatMoney(out, value);
  387. default:
  388. return out.append(value).append('?');
  389. }
  390. }
  391. StringBuffer & formatStatistic(StringBuffer & out, unsigned __int64 value, StatisticKind kind)
  392. {
  393. return formatStatistic(out, value, queryMeasure(kind));
  394. }
  395. //--------------------------------------------------------------------------------------------------------------------
  396. stat_type readStatisticValue(const char * cur, const char * * end, StatisticMeasure measure)
  397. {
  398. char * next = nullptr;
  399. stat_type value = strtoll(cur, &next, 10);
  400. switch (measure)
  401. {
  402. case SMeasureTimestampUs:
  403. {
  404. //ignore value and check if it is a formatted timestamp
  405. stat_type timestamp = extractTimestamp(cur, end);
  406. if (timestamp)
  407. return timestamp;
  408. //Assume that it is a single number
  409. break;
  410. }
  411. case SMeasureTimeNs:
  412. //Allow s, ms and us as scaling suffixes
  413. if (next[0] == 's')
  414. {
  415. value *= 1000000000;
  416. next++;
  417. }
  418. else if ((next[0] == 'm') && (next[1] == 's'))
  419. {
  420. value *= 1000000;
  421. next += 2;
  422. }
  423. else if ((next[0] == 'u') && (next[1] == 's'))
  424. {
  425. value *= 1000;
  426. next += 2;
  427. }
  428. else if ((next[0] == 'n') && (next[1] == 's'))
  429. {
  430. next += 2;
  431. }
  432. else
  433. {
  434. stat_type timestamp = extractTimeCollatable(cur, end);
  435. if (timestamp)
  436. return timestamp;
  437. //Assume that it is a single number
  438. }
  439. break;
  440. case SMeasureCount:
  441. case SMeasureSize:
  442. //Allow K, M, G as scaling suffixes
  443. if (next[0] == 'K')
  444. {
  445. value *= 0x400;
  446. next++;
  447. }
  448. else if (next[0] == 'M')
  449. {
  450. value *= 0x100000;
  451. next++;
  452. }
  453. else if (next[0] == 'G')
  454. {
  455. value *= 0x40000000;
  456. next++;
  457. }
  458. //Skip bytes marker
  459. if ((*next == 'b') || (*next == 'B'))
  460. next++;
  461. break;
  462. case SMeasurePercent:
  463. //MORE: Extend to allow fractional percentages
  464. //Allow % to mean a percentage - instead of ppm
  465. if (next[0] == '%')
  466. {
  467. value *= 10000;
  468. next++;
  469. }
  470. break;
  471. }
  472. if (end)
  473. *end = next;
  474. return value;
  475. }
  476. void validateScopeId(const char * idText)
  477. {
  478. StatsScopeId id;
  479. if (!id.setScopeText(idText))
  480. throw makeStringExceptionV(JLIBERR_UnexpectedValue, "'%s' does not appear to be a valid scope id", idText);
  481. }
  482. void validateScope(const char * scopeText)
  483. {
  484. StatsScopeId id;
  485. const char * cur = scopeText;
  486. for(;;)
  487. {
  488. if (!id.setScopeText(cur, &cur))
  489. throw makeStringExceptionV(JLIBERR_UnexpectedValue, "'%s' does not appear to be a valid scope id", cur);
  490. cur = strchr(cur, ':');
  491. if (!cur)
  492. return;
  493. cur++;
  494. }
  495. }
  496. //--------------------------------------------------------------------------------------------------------------------
  497. unsigned queryScopeDepth(const char * text)
  498. {
  499. if (!*text)
  500. return 0;
  501. unsigned depth = 1;
  502. for (;;)
  503. {
  504. switch (*text)
  505. {
  506. case 0:
  507. return depth;
  508. case ':':
  509. depth++;
  510. break;
  511. }
  512. text++;
  513. }
  514. }
  515. const char * queryScopeTail(const char * scope)
  516. {
  517. const char * colon = strrchr(scope, ':');
  518. if (colon)
  519. return colon+1;
  520. else
  521. return scope;
  522. }
  523. bool getParentScope(StringBuffer & parent, const char * scope)
  524. {
  525. const char * colon = strrchr(scope, ':');
  526. if (colon)
  527. {
  528. parent.append(colon-scope, scope);
  529. return true;
  530. }
  531. else
  532. return false;
  533. }
  534. void describeScope(StringBuffer & description, const char * scope)
  535. {
  536. if (!*scope)
  537. return;
  538. StatsScopeId id;
  539. for(;;)
  540. {
  541. id.extractScopeText(scope, &scope);
  542. id.describe(description);
  543. if (!*scope)
  544. return;
  545. description.append(": ");
  546. scope++;
  547. }
  548. }
  549. bool isParentScope(const char *parent, const char *scope)
  550. {
  551. const char *p = parent;
  552. const char *q = scope;
  553. while(*p && (*p==*q))
  554. {
  555. ++p;
  556. ++q;
  557. }
  558. if ((*p==0) && (*q==':' || *q==0))
  559. return true;
  560. return false;
  561. }
  562. const char * queryMeasurePrefix(StatisticMeasure measure)
  563. {
  564. switch (measure)
  565. {
  566. case SMeasureAll: return nullptr;
  567. case SMeasureTimeNs: return "Time";
  568. case SMeasureTimestampUs: return "When";
  569. case SMeasureCount: return "Num";
  570. case SMeasureSize: return "Size";
  571. case SMeasureLoad: return "Load";
  572. case SMeasureSkew: return "Skew";
  573. case SMeasureNode: return "Node";
  574. case SMeasurePercent: return "Per";
  575. case SMeasureIPV4: return "Ip";
  576. case SMeasureCycle: return "Cycle";
  577. case SMeasureEnum: return "";
  578. case SMeasureText: return "";
  579. case SMeasureBool: return "Is";
  580. case SMeasureId: return "Id";
  581. case SMeasureFilename: return "";
  582. case SMeasureCost: return "Cost";
  583. case SMeasureNone: return nullptr;
  584. default:
  585. return "Unknown";
  586. }
  587. }
  588. const char * queryMeasureName(StatisticMeasure measure)
  589. {
  590. return measureNames[measure];
  591. }
  592. StatisticMeasure queryMeasure(const char * measure, StatisticMeasure dft)
  593. {
  594. //MORE: Use a hash table??
  595. StatisticMeasure ret = (StatisticMeasure)matchString(measureNames, measure, SMeasureMax);
  596. if (ret != SMeasureMax)
  597. return ret;
  598. //Legacy support for an unusual statistic - pretend the sizes are in bytes instead of kb.
  599. if (streq(measure, "kb"))
  600. return SMeasureSize;
  601. for (unsigned i1=SMeasureAll+1; i1 < SMeasureMax; i1++)
  602. {
  603. const char * prefix = queryMeasurePrefix((StatisticMeasure)i1);
  604. if (strieq(measure, prefix))
  605. return (StatisticMeasure)i1;
  606. }
  607. return dft;
  608. }
  609. StatsMergeAction queryMergeMode(StatisticMeasure measure)
  610. {
  611. switch (measure)
  612. {
  613. case SMeasureTimeNs: return StatsMergeSum;
  614. case SMeasureTimestampUs: return StatsMergeKeepNonZero;
  615. case SMeasureCount: return StatsMergeSum;
  616. case SMeasureSize: return StatsMergeSum;
  617. case SMeasureLoad: return StatsMergeMax;
  618. case SMeasureSkew: return StatsMergeMax;
  619. case SMeasureNode: return StatsMergeKeepNonZero;
  620. case SMeasurePercent: return StatsMergeReplace;
  621. case SMeasureIPV4: return StatsMergeKeepNonZero;
  622. case SMeasureCycle: return StatsMergeSum;
  623. case SMeasureEnum: return StatsMergeKeepNonZero;
  624. case SMeasureText: return StatsMergeKeepNonZero;
  625. case SMeasureBool: return StatsMergeKeepNonZero;
  626. case SMeasureId: return StatsMergeKeepNonZero;
  627. case SMeasureFilename: return StatsMergeKeepNonZero;
  628. case SMeasureCost: return StatsMergeSum;
  629. default:
  630. #ifdef _DEBUG
  631. throwUnexpected();
  632. #else
  633. return StatsMergeSum;
  634. #endif
  635. }
  636. }
  637. extern jlib_decl StatsMergeAction queryMergeMode(StatisticKind kind)
  638. {
  639. //MORE: Optimize by looking up in the meta
  640. return queryMergeMode(queryMeasure(kind));
  641. }
  642. //--------------------------------------------------------------------------------------------------------------------
  643. #define BASE_NAMES(x, y) \
  644. #x #y, \
  645. #x "Min" # y, \
  646. #x "Max" # y, \
  647. #x "Avg" # y, \
  648. "Skew" # y, \
  649. "SkewMin" # y, \
  650. "SkewMax" # y, \
  651. "NodeMin" # y, \
  652. "NodeMax" # y,
  653. #define NAMES(x, y) \
  654. BASE_NAMES(x, y) \
  655. #x "Delta" # y, \
  656. #x "StdDev" #y,
  657. #define WHENNAMES(x, y) \
  658. BASE_NAMES(x, y) \
  659. "TimeDelta" # y, \
  660. "TimeStdDev" # y,
  661. #define BASE_TAGS(x, y) \
  662. "@" #x "Min" # y, \
  663. "@" #x "Max" # y, \
  664. "@" #x "Avg" # y, \
  665. "@Skew" # y, \
  666. "@SkewMin" # y, \
  667. "@SkewMax" # y, \
  668. "@NodeMin" # y, \
  669. "@NodeMax" # y,
  670. //Default tags nothing special overriden
  671. #define TAGS(x, y) \
  672. "@" #x #y, \
  673. BASE_TAGS(x, y) \
  674. "@" #x "Delta" # y, \
  675. "@" #x "StdDev" # y,
  676. //Define the tags for time items.
  677. #define WHENTAGS(x, y) \
  678. "@" #x #y, \
  679. BASE_TAGS(x, y) \
  680. "@TimeDelta" # y, \
  681. "@TimeStdDev" # y,
  682. #define CORESTAT(x, y, m) St##x##y, m, St##x##y, St##x##y, { NAMES(x, y) }, { TAGS(x, y) }
  683. #define STAT(x, y, m) CORESTAT(x, y, m)
  684. //--------------------------------------------------------------------------------------------------------------------
  685. //These are the macros to use to define the different entries in the stats meta table
  686. //#define TIMESTAT(y) STAT(Time, y, SMeasureTimeNs)
  687. #define TIMESTAT(y) St##Time##y, SMeasureTimeNs, St##Time##y, St##Cycle##y##Cycles, { NAMES(Time, y) }, { TAGS(Time, y) }
  688. #define WHENSTAT(y) St##When##y, SMeasureTimestampUs, St##When##y, St##When##y, { WHENNAMES(When, y) }, { WHENTAGS(When, y) }
  689. #define NUMSTAT(y) STAT(Num, y, SMeasureCount)
  690. #define SIZESTAT(y) STAT(Size, y, SMeasureSize)
  691. #define LOADSTAT(y) STAT(Load, y, SMeasureLoad)
  692. #define SKEWSTAT(y) STAT(Skew, y, SMeasureSkew)
  693. #define NODESTAT(y) STAT(Node, y, SMeasureNode)
  694. #define PERSTAT(y) STAT(Per, y, SMeasurePercent)
  695. #define IPV4STAT(y) STAT(IPV4, y, SMeasureIPV4)
  696. #define CYCLESTAT(y) St##Cycle##y##Cycles, SMeasureCycle, St##Time##y, St##Cycle##y##Cycles, { NAMES(Cycle, y##Cycles) }, { TAGS(Cycle, y##Cycles) }
  697. #define ENUMSTAT(y) STAT(Enum, y, SMeasureEnum)
  698. #define COSTSTAT(y) STAT(Cost, y, SMeasureCost)
  699. //--------------------------------------------------------------------------------------------------------------------
  700. class StatisticMeta
  701. {
  702. public:
  703. StatisticKind kind;
  704. StatisticMeasure measure;
  705. StatisticKind serializeKind;
  706. StatisticKind rawKind;
  707. const char * names[StNextModifier/StVariantScale];
  708. const char * tags[StNextModifier/StVariantScale];
  709. };
  710. //The order of entries in this table must match the order in the enumeration
  711. static const StatisticMeta statsMetaData[StMax] = {
  712. { StKindNone, SMeasureNone, StKindNone, StKindNone, { "none" }, { "@none" } },
  713. { StKindAll, SMeasureAll, StKindAll, StKindAll, { "all" }, { "@all" } },
  714. { WHENSTAT(GraphStarted) }, // Deprecated - use WhenStart
  715. { WHENSTAT(GraphFinished) }, // Deprecated - use WhenFinished
  716. { WHENSTAT(FirstRow) },
  717. { WHENSTAT(QueryStarted) }, // Deprecated - use WhenStart
  718. { WHENSTAT(QueryFinished) }, // Deprecated - use WhenFinished
  719. { WHENSTAT(Created) },
  720. { WHENSTAT(Compiled) },
  721. { WHENSTAT(WorkunitModified) },
  722. { TIMESTAT(Elapsed) },
  723. { TIMESTAT(LocalExecute) },
  724. { TIMESTAT(TotalExecute) },
  725. { TIMESTAT(Remaining) },
  726. { SIZESTAT(GeneratedCpp) },
  727. { SIZESTAT(PeakMemory) },
  728. { SIZESTAT(MaxRowSize) },
  729. { NUMSTAT(RowsProcessed) },
  730. { NUMSTAT(Slaves) },
  731. { NUMSTAT(Starts) },
  732. { NUMSTAT(Stops) },
  733. { NUMSTAT(IndexSeeks) },
  734. { NUMSTAT(IndexScans) },
  735. { NUMSTAT(IndexWildSeeks) },
  736. { NUMSTAT(IndexSkips) },
  737. { NUMSTAT(IndexNullSkips) },
  738. { NUMSTAT(IndexMerges) },
  739. { NUMSTAT(IndexMergeCompares) },
  740. { NUMSTAT(PreFiltered) },
  741. { NUMSTAT(PostFiltered) },
  742. { NUMSTAT(BlobCacheHits) },
  743. { NUMSTAT(LeafCacheHits) },
  744. { NUMSTAT(NodeCacheHits) },
  745. { NUMSTAT(BlobCacheAdds) },
  746. { NUMSTAT(LeafCacheAdds) },
  747. { NUMSTAT(NodeCacheAdds) },
  748. { NUMSTAT(PreloadCacheHits) },
  749. { NUMSTAT(PreloadCacheAdds) },
  750. { NUMSTAT(ServerCacheHits) },
  751. { NUMSTAT(IndexAccepted) },
  752. { NUMSTAT(IndexRejected) },
  753. { NUMSTAT(AtmostTriggered) },
  754. { NUMSTAT(DiskSeeks) },
  755. { NUMSTAT(Iterations) },
  756. { LOADSTAT(WhileSorting) },
  757. { NUMSTAT(LeftRows) },
  758. { NUMSTAT(RightRows) },
  759. { PERSTAT(Replicated) },
  760. { NUMSTAT(DiskRowsRead) },
  761. { NUMSTAT(IndexRowsRead) },
  762. { NUMSTAT(DiskAccepted) },
  763. { NUMSTAT(DiskRejected) },
  764. { TIMESTAT(Soapcall) },
  765. { TIMESTAT(FirstExecute) },
  766. { TIMESTAT(DiskReadIO) },
  767. { TIMESTAT(DiskWriteIO) },
  768. { SIZESTAT(DiskRead) },
  769. { SIZESTAT(DiskWrite) },
  770. { CYCLESTAT(DiskReadIO) },
  771. { CYCLESTAT(DiskWriteIO) },
  772. { NUMSTAT(DiskReads) },
  773. { NUMSTAT(DiskWrites) },
  774. { NUMSTAT(Spills) },
  775. { TIMESTAT(SpillElapsed) },
  776. { TIMESTAT(SortElapsed) },
  777. { NUMSTAT(Groups) },
  778. { NUMSTAT(GroupMax) },
  779. { SIZESTAT(SpillFile) },
  780. { CYCLESTAT(SpillElapsed) },
  781. { CYCLESTAT(SortElapsed) },
  782. { NUMSTAT(Strands) },
  783. { CYCLESTAT(TotalExecute) },
  784. { NUMSTAT(Executions) },
  785. { TIMESTAT(TotalNested) },
  786. { CYCLESTAT(LocalExecute) },
  787. { NUMSTAT(Compares) },
  788. { NUMSTAT(ScansPerRow) },
  789. { NUMSTAT(Allocations) },
  790. { NUMSTAT(AllocationScans) },
  791. { NUMSTAT(DiskRetries) },
  792. { CYCLESTAT(Elapsed) },
  793. { CYCLESTAT(Remaining) },
  794. { CYCLESTAT(Soapcall) },
  795. { CYCLESTAT(FirstExecute) },
  796. { CYCLESTAT(TotalNested) },
  797. { TIMESTAT(Generate) },
  798. { CYCLESTAT(Generate) },
  799. { WHENSTAT(Started) },
  800. { WHENSTAT(Finished) },
  801. { NUMSTAT(AnalyseExprs) },
  802. { NUMSTAT(TransformExprs) },
  803. { NUMSTAT(UniqueAnalyseExprs) },
  804. { NUMSTAT(UniqueTransformExprs) },
  805. { NUMSTAT(DuplicateKeys) },
  806. { NUMSTAT(AttribsProcessed) },
  807. { NUMSTAT(AttribsSimplified) },
  808. { NUMSTAT(AttribsFromCache) },
  809. { NUMSTAT(SmartJoinDegradedToLocal) },
  810. { NUMSTAT(SmartJoinSlavesDegradedToStd) },
  811. { NUMSTAT(AttribsSimplifiedTooComplex) },
  812. { NUMSTAT(SysContextSwitches) },
  813. { TIMESTAT(OsUser) },
  814. { TIMESTAT(OsSystem) },
  815. { TIMESTAT(OsTotal) },
  816. { CYCLESTAT(OsUser) },
  817. { CYCLESTAT(OsSystem) },
  818. { CYCLESTAT(OsTotal) },
  819. { NUMSTAT(ContextSwitches) },
  820. { TIMESTAT(User) },
  821. { TIMESTAT(System) },
  822. { TIMESTAT(Total) },
  823. { CYCLESTAT(User) },
  824. { CYCLESTAT(System) },
  825. { CYCLESTAT(Total) },
  826. { SIZESTAT(OsDiskRead) },
  827. { SIZESTAT(OsDiskWrite) },
  828. { TIMESTAT(Blocked) },
  829. { CYCLESTAT(Blocked) },
  830. { COSTSTAT(Execute) },
  831. { SIZESTAT(AgentReply) },
  832. { TIMESTAT(AgentWait) },
  833. { CYCLESTAT(AgentWait) },
  834. { COSTSTAT(FileAccess) },
  835. { NUMSTAT(Pods) }
  836. };
  837. //Is a 0 value likely, and useful to be reported if it does happen to be zero?
  838. bool includeStatisticIfZero(StatisticKind kind)
  839. {
  840. switch (kind)
  841. {
  842. case StNumRowsProcessed:
  843. case StNumIterations:
  844. case StNumIndexSeeks:
  845. case StNumDuplicateKeys:
  846. return true;
  847. }
  848. return false;
  849. }
  850. //--------------------------------------------------------------------------------------------------------------------
  851. StatisticMeasure queryMeasure(StatisticKind kind)
  852. {
  853. unsigned variant = queryStatsVariant(kind);
  854. switch (variant)
  855. {
  856. case StSkew:
  857. case StSkewMin:
  858. case StSkewMax:
  859. return SMeasureSkew;
  860. case StNodeMin:
  861. case StNodeMax:
  862. return SMeasureNode;
  863. case StDeltaX:
  864. case StStdDevX:
  865. {
  866. StatisticMeasure measure = queryMeasure((StatisticKind)(kind & StKindMask));
  867. switch (measure)
  868. {
  869. case SMeasureTimestampUs:
  870. return SMeasureTimeNs;
  871. default:
  872. return measure;
  873. }
  874. break;
  875. }
  876. }
  877. StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
  878. if (rawkind >= StKindNone && rawkind < StMax)
  879. return statsMetaData[rawkind].measure;
  880. return SMeasureNone;
  881. }
  882. const char * queryStatisticName(StatisticKind kind)
  883. {
  884. StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
  885. unsigned variant = (kind / StVariantScale);
  886. dbgassertex(variant < (StNextModifier/StVariantScale));
  887. if (rawkind >= StKindNone && rawkind < StMax)
  888. return statsMetaData[rawkind].names[variant];
  889. return "Unknown";
  890. }
  891. unsigned __int64 convertMeasure(StatisticMeasure from, StatisticMeasure to, unsigned __int64 value)
  892. {
  893. if (from == to)
  894. return value;
  895. if ((from == SMeasureCycle) && (to == SMeasureTimeNs))
  896. return cycle_to_nanosec(value);
  897. if ((from == SMeasureTimeNs) && (to == SMeasureCycle))
  898. return nanosec_to_cycle(value);
  899. if ((from == SMeasureTimestampUs) && (to == SMeasureTimeNs))
  900. return value * 1000;
  901. if ((from == SMeasureTimeNs) && (to == SMeasureTimestampUs))
  902. return value / 1000;
  903. #ifdef _DEBUG
  904. throwUnexpected();
  905. #else
  906. return value;
  907. #endif
  908. }
  909. unsigned __int64 convertMeasure(StatisticKind from, StatisticKind to, unsigned __int64 value)
  910. {
  911. if (from == to)
  912. return value;
  913. return convertMeasure(queryMeasure(from), queryMeasure(to), value);
  914. }
  915. static unsigned __int64 convertSumMeasure(StatisticKind from, StatisticKind to, double value)
  916. {
  917. if (from == to)
  918. return value;
  919. return convertMeasure(queryMeasure(from), queryMeasure(to), value);
  920. }
  921. static double convertSquareMeasure(StatisticMeasure from, StatisticMeasure to, double value)
  922. {
  923. if (from == to)
  924. return value;
  925. //Coded to a avoid overflow of unsigned __int64 in cycle_to_nanosec etc.
  926. const unsigned __int64 largeValue = 1000000000;
  927. double scale;
  928. if ((from == SMeasureCycle) && (to == SMeasureTimeNs))
  929. scale = (double)cycle_to_nanosec(largeValue) / (double)largeValue;
  930. else if ((from == SMeasureTimeNs) && (to == SMeasureCycle))
  931. scale = (double)nanosec_to_cycle(largeValue) / (double)largeValue;
  932. else
  933. {
  934. #ifdef _DEBUG
  935. throwUnexpected();
  936. #else
  937. scale = 1.0;
  938. #endif
  939. }
  940. return value * scale * scale;
  941. }
  942. static double convertSquareMeasure(StatisticKind from, StatisticKind to, double value)
  943. {
  944. return convertSquareMeasure(queryMeasure(from), queryMeasure(to), value);
  945. }
  946. static StatisticKind querySerializedKind(StatisticKind kind)
  947. {
  948. StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
  949. if (rawkind >= StMax)
  950. return kind;
  951. StatisticKind serialKind = statsMetaData[rawkind].serializeKind;
  952. return (StatisticKind)(serialKind | (kind & ~StKindMask));
  953. }
  954. static StatisticKind queryRawKind(StatisticKind kind)
  955. {
  956. StatisticKind basekind = (StatisticKind)(kind & StKindMask);
  957. if (basekind >= StMax)
  958. return kind;
  959. StatisticKind rawKind = statsMetaData[basekind].rawKind;
  960. return (StatisticKind)(rawKind | (kind & ~StKindMask));
  961. }
  962. //--------------------------------------------------------------------------------------------------------------------
  963. void queryLongStatisticName(StringBuffer & out, StatisticKind kind)
  964. {
  965. out.append(queryStatisticName(kind));
  966. }
  967. //--------------------------------------------------------------------------------------------------------------------
  968. const char * queryTreeTag(StatisticKind kind)
  969. {
  970. StatisticKind rawkind = (StatisticKind)(kind & StKindMask);
  971. unsigned variant = (kind / StVariantScale);
  972. dbgassertex(variant < (StNextModifier/StVariantScale));
  973. if (rawkind >= StKindNone && rawkind < StMax)
  974. return statsMetaData[rawkind].tags[variant];
  975. return "@Unknown";
  976. }
  977. //--------------------------------------------------------------------------------------------------------------------
  978. StatisticKind queryStatisticKind(const char * search, StatisticKind dft)
  979. {
  980. if (!search)
  981. return dft;
  982. if (streq(search, "*"))
  983. return StKindAll;
  984. //Slow - should use a hash table....
  985. for (unsigned variant=0; variant < StNextModifier; variant += StVariantScale)
  986. {
  987. for (unsigned i=0; i < StMax; i++)
  988. {
  989. StatisticKind kind = (StatisticKind)(i+variant);
  990. const char * shortName = queryStatisticName(kind);
  991. if (shortName && strieq(shortName, search))
  992. return kind;
  993. }
  994. }
  995. return dft;
  996. }
  997. //--------------------------------------------------------------------------------------------------------------------
  998. const char * queryCreatorTypeName(StatisticCreatorType sct)
  999. {
  1000. return creatorTypeNames[sct];
  1001. }
  1002. StatisticCreatorType queryCreatorType(const char * sct, StatisticCreatorType dft)
  1003. {
  1004. //MORE: Use a hash table??
  1005. return (StatisticCreatorType)matchString(creatorTypeNames, sct, dft);
  1006. }
  1007. //--------------------------------------------------------------------------------------------------------------------
  1008. const char * queryScopeTypeName(StatisticScopeType sst)
  1009. {
  1010. return scopeTypeNames[sst];
  1011. }
  1012. extern jlib_decl StatisticScopeType queryScopeType(const char * sst, StatisticScopeType dft)
  1013. {
  1014. //MORE: Use a hash table??
  1015. return (StatisticScopeType)matchString(scopeTypeNames, sst, dft);
  1016. }
  1017. //--------------------------------------------------------------------------------------------------------------------
  1018. inline void mergeUpdate(StatisticMeasure measure, unsigned __int64 & value, const unsigned __int64 otherValue)
  1019. {
  1020. switch (measure)
  1021. {
  1022. case SMeasureTimeNs:
  1023. case SMeasureCount:
  1024. case SMeasureSize:
  1025. case SMeasureLoad:
  1026. case SMeasureSkew:
  1027. case SMeasureCycle:
  1028. value += otherValue;
  1029. break;
  1030. case SMeasureTimestampUs:
  1031. if (otherValue && otherValue < value)
  1032. value = otherValue;
  1033. break;
  1034. }
  1035. }
  1036. unsigned __int64 mergeStatistic(StatisticMeasure measure, unsigned __int64 value, unsigned __int64 otherValue)
  1037. {
  1038. mergeUpdate(measure, value, otherValue);
  1039. return value;
  1040. }
  1041. unsigned __int64 mergeStatisticValue(unsigned __int64 prevValue, unsigned __int64 newValue, StatsMergeAction mergeAction)
  1042. {
  1043. switch (mergeAction)
  1044. {
  1045. case StatsMergeKeepNonZero:
  1046. if (prevValue)
  1047. return prevValue;
  1048. return newValue;
  1049. case StatsMergeAppend:
  1050. case StatsMergeReplace:
  1051. return newValue;
  1052. case StatsMergeSum:
  1053. return prevValue + newValue;
  1054. case StatsMergeMin:
  1055. if (prevValue > newValue)
  1056. return newValue;
  1057. else
  1058. return prevValue;
  1059. case StatsMergeMax:
  1060. if (prevValue < newValue)
  1061. return newValue;
  1062. else
  1063. return prevValue;
  1064. default:
  1065. #ifdef _DEBUG
  1066. throwUnexpected();
  1067. #else
  1068. return newValue;
  1069. #endif
  1070. }
  1071. }
  1072. //--------------------------------------------------------------------------------------------------------------------
  1073. class CComponentStatistics
  1074. {
  1075. protected:
  1076. StringAttr creator;
  1077. byte creatorDepth;
  1078. byte scopeDepth;
  1079. // StatisticArray stats;
  1080. };
  1081. //--------------------------------------------------------------------------------------------------------------------
  1082. static int compareUnsigned(unsigned const * left, unsigned const * right)
  1083. {
  1084. return (*left < *right) ? -1 : (*left > *right) ? +1 : 0;
  1085. }
  1086. void StatisticsMapping::createMappings()
  1087. {
  1088. //Possibly not needed, but sort the kinds, so that it is easy to merge/stream the results out in the correct order.
  1089. indexToKind.sort(compareUnsigned);
  1090. //Provide mappings to all statistics to map them to the "unknown" bin by default
  1091. for (unsigned i=0; i < StMax; i++)
  1092. kindToIndex.append(numStatistics());
  1093. ForEachItemIn(i2, indexToKind)
  1094. {
  1095. unsigned kind = indexToKind.item(i2);
  1096. kindToIndex.replace(i2, kind);
  1097. }
  1098. }
  1099. const StatisticsMapping allStatistics(StKindAll);
  1100. const StatisticsMapping heapStatistics({StNumAllocations, StNumAllocationScans});
  1101. const StatisticsMapping diskLocalStatistics({StCycleDiskReadIOCycles, StSizeDiskRead, StNumDiskReads, StCycleDiskWriteIOCycles, StSizeDiskWrite, StNumDiskWrites, StNumDiskRetries});
  1102. const StatisticsMapping diskRemoteStatistics({StTimeDiskReadIO, StSizeDiskRead, StNumDiskReads, StTimeDiskWriteIO, StSizeDiskWrite, StNumDiskWrites, StNumDiskRetries});
  1103. const StatisticsMapping diskReadRemoteStatistics({StTimeDiskReadIO, StSizeDiskRead, StNumDiskReads, StNumDiskRetries, StCycleDiskReadIOCycles});
  1104. const StatisticsMapping diskWriteRemoteStatistics({StTimeDiskWriteIO, StSizeDiskWrite, StNumDiskWrites, StNumDiskRetries, StCycleDiskWriteIOCycles});
  1105. //--------------------------------------------------------------------------------------------------------------------
  1106. StringBuffer & StatisticValueFilter::describe(StringBuffer & out) const
  1107. {
  1108. out.append(queryStatisticName(kind));
  1109. if (minValue == maxValue)
  1110. out.append("=").append(minValue);
  1111. else
  1112. out.append("=").append(minValue).append("..").append(maxValue);
  1113. return out;
  1114. }
  1115. //--------------------------------------------------------------------------------------------------------------------
  1116. class Statistic
  1117. {
  1118. public:
  1119. Statistic(StatisticKind _kind, unsigned __int64 _value) : kind(_kind), value(_value)
  1120. {
  1121. }
  1122. Statistic(MemoryBuffer & in, unsigned version)
  1123. {
  1124. unsigned _kind;
  1125. in.read(_kind);
  1126. kind = (StatisticKind)_kind;
  1127. in.read(value);
  1128. }
  1129. StatisticKind queryKind() const
  1130. {
  1131. return kind;
  1132. }
  1133. unsigned __int64 queryValue() const
  1134. {
  1135. return value;
  1136. }
  1137. void merge(unsigned __int64 otherValue)
  1138. {
  1139. mergeUpdate(queryMeasure(kind), value, otherValue);
  1140. }
  1141. void serialize(MemoryBuffer & out) const
  1142. {
  1143. //MORE: Could compress - e.g., store as a packed integers
  1144. out.append((unsigned)kind);
  1145. out.append(value);
  1146. }
  1147. void mergeInto(IStatisticGatherer & target) const
  1148. {
  1149. StatsMergeAction mergeAction = queryMergeMode(kind);
  1150. target.updateStatistic(kind, value, mergeAction);
  1151. }
  1152. StringBuffer & toXML(StringBuffer &out) const
  1153. {
  1154. return out.append(" <Stat name=\"").append(queryStatisticName(kind)).append("\" value=\"").append(value).append("\"/>\n");
  1155. }
  1156. public:
  1157. StatisticKind kind;
  1158. unsigned __int64 value;
  1159. };
  1160. //--------------------------------------------------------------------------------------------------------------------
  1161. StringBuffer & StatsScopeId::getScopeText(StringBuffer & out) const
  1162. {
  1163. switch (scopeType)
  1164. {
  1165. case SSTgraph:
  1166. return out.append(GraphScopePrefix).append(id);
  1167. case SSTsubgraph:
  1168. return out.append(SubGraphScopePrefix).append(id);
  1169. case SSTactivity:
  1170. return out.append(ActivityScopePrefix).append(id);
  1171. case SSTedge:
  1172. return out.append(EdgeScopePrefix).append(id).append("_").append(extra);
  1173. case SSTfunction:
  1174. return out.append(FunctionScopePrefix).append(name);
  1175. case SSTworkflow:
  1176. return out.append(WorkflowScopePrefix).append(id);
  1177. case SSTchildgraph:
  1178. return out.append(ChildGraphScopePrefix).append(id);
  1179. case SSTfile:
  1180. return out.append(FileScopePrefix).append(name);
  1181. case SSTchannel:
  1182. return out.append(ChannelScopePrefix).append(id);
  1183. case SSTunknown:
  1184. return out.append(name);
  1185. default:
  1186. #ifdef _DEBUG
  1187. throwUnexpected();
  1188. #endif
  1189. return out.append("????").append(id);
  1190. }
  1191. }
  1192. unsigned StatsScopeId::getHash() const
  1193. {
  1194. switch (scopeType)
  1195. {
  1196. case SSTfunction:
  1197. case SSTunknown:
  1198. return hashc((const byte *)name.get(), strlen(name), (unsigned)scopeType);
  1199. default:
  1200. return hashc((const byte *)&id, sizeof(id), (unsigned)scopeType);
  1201. }
  1202. }
  1203. bool StatsScopeId::isWildcard() const
  1204. {
  1205. return (id == 0) && (extra == 0) && !name;
  1206. }
  1207. int StatsScopeId::compare(const StatsScopeId & other) const
  1208. {
  1209. if (scopeType != other.scopeType)
  1210. return scopePriority[scopeType] - scopePriority[other.scopeType];
  1211. if (id != other.id)
  1212. return (int)(id - other.id);
  1213. if (extra != other.extra)
  1214. return (int)(extra - other.extra);
  1215. if (name && other.name)
  1216. return strcmp(name, other.name);
  1217. if (name)
  1218. return +1;
  1219. if (other.name)
  1220. return -1;
  1221. return 0;
  1222. }
  1223. void StatsScopeId::describe(StringBuffer & description) const
  1224. {
  1225. const char * name = queryScopeTypeName(scopeType);
  1226. description.append((char)toupper(*name)).append(name+1);
  1227. switch (scopeType)
  1228. {
  1229. case SSTgraph:
  1230. description.append(" graph").append(id);
  1231. break;
  1232. case SSTsubgraph:
  1233. case SSTactivity:
  1234. case SSTworkflow:
  1235. case SSTchildgraph:
  1236. case SSTchannel:
  1237. description.append(' ').append(id);
  1238. break;
  1239. case SSTedge:
  1240. description.append(' ').append(id).append(',').append(extra);
  1241. break;
  1242. case SSTfile:
  1243. case SSTfunction:
  1244. description.append(' ').append(name);
  1245. break;
  1246. default:
  1247. throwUnexpected();
  1248. break;
  1249. }
  1250. }
  1251. bool StatsScopeId::matches(const StatsScopeId & other) const
  1252. {
  1253. return (scopeType == other.scopeType) && (id == other.id) && (extra == other.extra) && strsame(name, other.name);
  1254. }
  1255. unsigned StatsScopeId::queryActivity() const
  1256. {
  1257. switch (scopeType)
  1258. {
  1259. case SSTactivity:
  1260. case SSTedge:
  1261. return id;
  1262. default:
  1263. return 0;
  1264. }
  1265. }
  1266. void StatsScopeId::deserialize(MemoryBuffer & in, unsigned version)
  1267. {
  1268. byte scopeTypeByte;
  1269. in.read(scopeTypeByte);
  1270. scopeType = (StatisticScopeType)scopeTypeByte;
  1271. switch (scopeType)
  1272. {
  1273. case SSTgraph:
  1274. case SSTsubgraph:
  1275. case SSTactivity:
  1276. case SSTworkflow:
  1277. case SSTchildgraph:
  1278. case SSTchannel:
  1279. in.read(id);
  1280. break;
  1281. case SSTedge:
  1282. in.read(id);
  1283. in.read(extra);
  1284. break;
  1285. case SSTfile:
  1286. case SSTfunction:
  1287. in.read(name);
  1288. break;
  1289. default:
  1290. throwUnexpected();
  1291. break;
  1292. }
  1293. }
  1294. void StatsScopeId::serialize(MemoryBuffer & out) const
  1295. {
  1296. out.append((byte)scopeType);
  1297. switch (scopeType)
  1298. {
  1299. case SSTgraph:
  1300. case SSTsubgraph:
  1301. case SSTactivity:
  1302. case SSTworkflow:
  1303. case SSTchildgraph:
  1304. case SSTchannel:
  1305. out.append(id);
  1306. break;
  1307. case SSTedge:
  1308. out.append(id);
  1309. out.append(extra);
  1310. break;
  1311. case SSTfile:
  1312. case SSTfunction:
  1313. out.append(name);
  1314. break;
  1315. default:
  1316. throwUnexpected();
  1317. break;
  1318. }
  1319. }
  1320. void StatsScopeId::setId(StatisticScopeType _scopeType, unsigned _id, unsigned _extra)
  1321. {
  1322. scopeType = _scopeType;
  1323. id = _id;
  1324. extra = _extra;
  1325. }
  1326. bool StatsScopeId::setScopeText(const char * text, const char * * _next)
  1327. {
  1328. char * * next = (char * *)_next;
  1329. switch (*text)
  1330. {
  1331. case ActivityScopePrefix[0]:
  1332. if (MATCHES_CONST_PREFIX(text, ActivityScopePrefix))
  1333. {
  1334. if (isdigit(text[strlen(ActivityScopePrefix)]))
  1335. {
  1336. unsigned id = strtoul(text + strlen(ActivityScopePrefix), next, 10);
  1337. setActivityId(id);
  1338. return true;
  1339. }
  1340. }
  1341. break;
  1342. case GraphScopePrefix[0]:
  1343. if (MATCHES_CONST_PREFIX(text, GraphScopePrefix))
  1344. {
  1345. if (isdigit(text[strlen(GraphScopePrefix)]))
  1346. {
  1347. unsigned id = strtoul(text + strlen(GraphScopePrefix), next, 10);
  1348. setId(SSTgraph, id);
  1349. return true;
  1350. }
  1351. }
  1352. break;
  1353. case SubGraphScopePrefix[0]:
  1354. if (MATCHES_CONST_PREFIX(text, SubGraphScopePrefix))
  1355. {
  1356. if (isdigit(text[strlen(SubGraphScopePrefix)]))
  1357. {
  1358. unsigned id = strtoul(text + strlen(SubGraphScopePrefix), next, 10);
  1359. setSubgraphId(id);
  1360. return true;
  1361. }
  1362. }
  1363. break;
  1364. case EdgeScopePrefix[0]:
  1365. if (MATCHES_CONST_PREFIX(text, EdgeScopePrefix))
  1366. {
  1367. const char * underscore = strchr(text, '_');
  1368. if (!underscore || !isdigit(underscore[1]))
  1369. return false;
  1370. unsigned id1 = atoi(text + strlen(EdgeScopePrefix));
  1371. unsigned id2 = strtoul(underscore+1, next, 10);
  1372. setEdgeId(id1, id2);
  1373. return true;
  1374. }
  1375. break;
  1376. case FunctionScopePrefix[0]:
  1377. if (MATCHES_CONST_PREFIX(text, FunctionScopePrefix))
  1378. {
  1379. setFunctionId(text+ strlen(FunctionScopePrefix));
  1380. if (_next)
  1381. *_next = text + strlen(text);
  1382. return true;
  1383. }
  1384. break;
  1385. case FileScopePrefix[0]:
  1386. if (MATCHES_CONST_PREFIX(text, FileScopePrefix))
  1387. {
  1388. setFileId(text+strlen(FileScopePrefix));
  1389. if (_next)
  1390. *_next = text + strlen(text);
  1391. return true;
  1392. }
  1393. break;
  1394. case WorkflowScopePrefix[0]:
  1395. if (MATCHES_CONST_PREFIX(text, WorkflowScopePrefix) && isdigit(text[strlen(WorkflowScopePrefix)]))
  1396. {
  1397. setWorkflowId(strtoul(text+ strlen(WorkflowScopePrefix), next, 10));
  1398. return true;
  1399. }
  1400. break;
  1401. case ChildGraphScopePrefix[0]:
  1402. if (MATCHES_CONST_PREFIX(text, ChildGraphScopePrefix))
  1403. {
  1404. setChildGraphId(strtoul(text+ strlen(ChildGraphScopePrefix), next, 10));
  1405. return true;
  1406. }
  1407. break;
  1408. case ChannelScopePrefix[0]:
  1409. if (MATCHES_CONST_PREFIX(text, ChannelScopePrefix) && isdigit(text[strlen(ChannelScopePrefix)]))
  1410. {
  1411. setChannelId(strtoul(text+ strlen(ChannelScopePrefix), next, 10));
  1412. return true;
  1413. }
  1414. break;
  1415. case '\0':
  1416. setId(SSTglobal, 0);
  1417. return true;
  1418. }
  1419. return false;
  1420. }
  1421. bool StatsScopeId::extractScopeText(const char * text, const char * * next)
  1422. {
  1423. if (setScopeText(text, next))
  1424. return true;
  1425. scopeType = SSTunknown;
  1426. const char * end = strchr(text, ':');
  1427. if (end)
  1428. {
  1429. name.set(text, end-text);
  1430. if (next)
  1431. *next = end;
  1432. }
  1433. else
  1434. {
  1435. name.set(text);
  1436. if (next)
  1437. *next = text + strlen(text);
  1438. }
  1439. return false;
  1440. }
  1441. void StatsScopeId::setActivityId(unsigned _id)
  1442. {
  1443. setId(SSTactivity, _id);
  1444. }
  1445. void StatsScopeId::setEdgeId(unsigned _id, unsigned _output)
  1446. {
  1447. setId(SSTedge, _id, _output);
  1448. }
  1449. void StatsScopeId::setSubgraphId(unsigned _id)
  1450. {
  1451. setId(SSTsubgraph, _id);
  1452. }
  1453. void StatsScopeId::setFunctionId(const char * _name)
  1454. {
  1455. scopeType = SSTfunction;
  1456. name.set(_name);
  1457. }
  1458. void StatsScopeId::setFileId(const char * _name)
  1459. {
  1460. scopeType = SSTfile;
  1461. name.set(_name);
  1462. }
  1463. void StatsScopeId::setChannelId(unsigned _id)
  1464. {
  1465. setId(SSTchannel, _id);
  1466. }
  1467. void StatsScopeId::setWorkflowId(unsigned _id)
  1468. {
  1469. setId(SSTworkflow, _id);
  1470. }
  1471. void StatsScopeId::setChildGraphId(unsigned _id)
  1472. {
  1473. setId(SSTchildgraph, _id);
  1474. }
  1475. //--------------------------------------------------------------------------------------------------------------------
  1476. enum
  1477. {
  1478. SCroot,
  1479. SCintermediate,
  1480. SCleaf,
  1481. };
  1482. class CStatisticCollection;
  1483. static CStatisticCollection * deserializeCollection(CStatisticCollection * parent, MemoryBuffer & in, unsigned version);
  1484. //MORE: Create an implementation with no children
  1485. typedef StructArrayOf<Statistic> StatsArray;
  1486. class CollectionHashTable : public SuperHashTableOf<CStatisticCollection, StatsScopeId>
  1487. {
  1488. public:
  1489. ~CollectionHashTable() { _releaseAll(); }
  1490. virtual void onAdd(void *et);
  1491. virtual void onRemove(void *et);
  1492. virtual unsigned getHashFromElement(const void *et) const;
  1493. virtual unsigned getHashFromFindParam(const void *fp) const;
  1494. virtual const void * getFindParam(const void *et) const;
  1495. virtual bool matchesFindParam(const void *et, const void *key, unsigned fphash) const;
  1496. virtual bool matchesElement(const void *et, const void *searchET) const;
  1497. };
  1498. typedef IArrayOf<CStatisticCollection> CollectionArray;
  1499. static int compareCollection(IInterface * const * pl, IInterface * const *pr);
  1500. class SortedCollectionIterator : public ArrayIIteratorOf<IArrayOf<IStatisticCollection>, IStatisticCollection, IStatisticCollectionIterator>
  1501. {
  1502. IArrayOf<IStatisticCollection> elems;
  1503. public:
  1504. SortedCollectionIterator(IStatisticCollectionIterator &iter) : ArrayIIteratorOf<IArrayOf<IStatisticCollection>, IStatisticCollection, IStatisticCollectionIterator>(elems)
  1505. {
  1506. ForEach(iter)
  1507. elems.append(iter.get());
  1508. elems.sort(compareCollection);
  1509. }
  1510. };
  1511. class CStatisticCollection : public CInterfaceOf<IStatisticCollection>
  1512. {
  1513. friend class CollectionHashTable;
  1514. public:
  1515. CStatisticCollection(CStatisticCollection * _parent, const StatsScopeId & _id) : id(_id), parent(_parent)
  1516. {
  1517. }
  1518. CStatisticCollection(CStatisticCollection * _parent, MemoryBuffer & in, unsigned version) : parent(_parent)
  1519. {
  1520. id.deserialize(in, version);
  1521. unsigned numStats;
  1522. in.read(numStats);
  1523. stats.ensureCapacity(numStats);
  1524. while (numStats-- > 0)
  1525. {
  1526. Statistic next (in, version);
  1527. stats.append(next);
  1528. }
  1529. unsigned numChildren;
  1530. in.read(numChildren);
  1531. children.ensure(numChildren);
  1532. while (numChildren-- > 0)
  1533. {
  1534. CStatisticCollection * next = deserializeCollection(this, in, version);
  1535. children.add(*next);
  1536. }
  1537. }
  1538. virtual byte getCollectionType() const { return SCintermediate; }
  1539. //interface IStatisticCollection:
  1540. virtual StringBuffer &toXML(StringBuffer &out) const override;
  1541. virtual StatisticScopeType queryScopeType() const override
  1542. {
  1543. return id.queryScopeType();
  1544. }
  1545. virtual unsigned __int64 queryWhenCreated() const override
  1546. {
  1547. if (parent)
  1548. return parent->queryWhenCreated();
  1549. return 0;
  1550. }
  1551. virtual StringBuffer & getScope(StringBuffer & str) const override
  1552. {
  1553. return id.getScopeText(str);
  1554. }
  1555. virtual StringBuffer & getFullScope(StringBuffer & str) const override
  1556. {
  1557. if (parent)
  1558. {
  1559. parent->getFullScope(str);
  1560. str.append(':');
  1561. }
  1562. id.getScopeText(str);
  1563. return str;
  1564. }
  1565. virtual unsigned __int64 queryStatistic(StatisticKind kind) const override
  1566. {
  1567. ForEachItemIn(i, stats)
  1568. {
  1569. const Statistic & cur = stats.item(i);
  1570. if (cur.kind == kind)
  1571. return cur.value;
  1572. }
  1573. return 0;
  1574. }
  1575. virtual bool getStatistic(StatisticKind kind, unsigned __int64 & value) const override
  1576. {
  1577. ForEachItemIn(i, stats)
  1578. {
  1579. const Statistic & cur = stats.item(i);
  1580. if (cur.kind == kind)
  1581. {
  1582. value = cur.value;
  1583. return true;
  1584. }
  1585. }
  1586. return false;
  1587. }
  1588. virtual unsigned getNumStatistics() const override
  1589. {
  1590. return stats.ordinality();
  1591. }
  1592. virtual void getStatistic(StatisticKind & kind, unsigned __int64 & value, unsigned idx) const override
  1593. {
  1594. const Statistic & cur = stats.item(idx);
  1595. kind = cur.kind;
  1596. value = cur.value;
  1597. }
  1598. virtual IStatisticCollectionIterator & getScopes(const char * filter, bool sorted) override
  1599. {
  1600. assertex(!filter);
  1601. Owned<IStatisticCollectionIterator> hashIter = new SuperHashIIteratorOf<IStatisticCollection, IStatisticCollectionIterator, false>(children);
  1602. if (!sorted)
  1603. return *hashIter.getClear();
  1604. return * new SortedCollectionIterator(*hashIter);
  1605. }
  1606. virtual void getMinMaxScope(IStringVal & minValue, IStringVal & maxValue, StatisticScopeType searchScopeType) const override
  1607. {
  1608. if (id.queryScopeType() == searchScopeType)
  1609. {
  1610. const char * curMin = minValue.str();
  1611. const char * curMax = maxValue.str();
  1612. StringBuffer name;
  1613. id.getScopeText(name);
  1614. if (!curMin || !*curMin || strcmp(name.str(), curMin) < 0)
  1615. minValue.set(name.str());
  1616. if (!curMax || strcmp(name.str(), curMax) > 0)
  1617. maxValue.set(name.str());
  1618. }
  1619. for (auto & curChild : children)
  1620. curChild.getMinMaxScope(minValue, maxValue, searchScopeType);
  1621. }
  1622. virtual void getMinMaxActivity(unsigned & minValue, unsigned & maxValue) const override
  1623. {
  1624. unsigned activityId = id.queryActivity();
  1625. if (activityId)
  1626. {
  1627. if ((minValue == 0) || (activityId < minValue))
  1628. minValue = activityId;
  1629. if (activityId > maxValue)
  1630. maxValue = activityId;
  1631. }
  1632. SuperHashIteratorOf<CStatisticCollection> iter(children, false);
  1633. for (iter.first(); iter.isValid(); iter.next())
  1634. iter.query().getMinMaxActivity(minValue, maxValue);
  1635. }
  1636. //other public interface functions
  1637. void addStatistic(StatisticKind kind, unsigned __int64 value)
  1638. {
  1639. Statistic s(kind, value);
  1640. stats.append(s);
  1641. }
  1642. void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
  1643. {
  1644. if (mergeAction != StatsMergeAppend)
  1645. {
  1646. ForEachItemIn(i, stats)
  1647. {
  1648. Statistic & cur = stats.element(i);
  1649. if (cur.kind == kind)
  1650. {
  1651. cur.value = mergeStatisticValue(cur.value, value, mergeAction);
  1652. return;
  1653. }
  1654. }
  1655. }
  1656. Statistic s(kind, value);
  1657. stats.append(s);
  1658. }
  1659. CStatisticCollection * ensureSubScope(const StatsScopeId & search, bool hasChildren)
  1660. {
  1661. //Once the CStatisicCollection is created it should not be replaced - so that returned pointers remain valid.
  1662. CStatisticCollection * match = children.find(&search);
  1663. if (match)
  1664. return match;
  1665. CStatisticCollection * ret = new CStatisticCollection(this, search);
  1666. children.add(*ret);
  1667. return ret;
  1668. }
  1669. virtual void serialize(MemoryBuffer & out) const
  1670. {
  1671. out.append(getCollectionType());
  1672. id.serialize(out);
  1673. out.append(stats.ordinality());
  1674. ForEachItemIn(iStat, stats)
  1675. stats.item(iStat).serialize(out);
  1676. out.append(children.ordinality());
  1677. SuperHashIteratorOf<CStatisticCollection> iter(children, false);
  1678. for (iter.first(); iter.isValid(); iter.next())
  1679. iter.query().serialize(out);
  1680. }
  1681. inline const StatsScopeId & queryScopeId() const { return id; }
  1682. virtual void mergeInto(IStatisticGatherer & target) const
  1683. {
  1684. StatsOptScope block(target, id);
  1685. ForEachItemIn(iStat, stats)
  1686. stats.item(iStat).mergeInto(target);
  1687. for (auto const & cur : children)
  1688. cur.mergeInto(target);
  1689. }
  1690. private:
  1691. StatsScopeId id;
  1692. CStatisticCollection * parent;
  1693. protected:
  1694. CollectionHashTable children;
  1695. StatsArray stats;
  1696. };
  1697. StringBuffer &CStatisticCollection::toXML(StringBuffer &out) const
  1698. {
  1699. out.append("<Scope id=\"");
  1700. id.getScopeText(out).append("\">\n");
  1701. if (stats.ordinality())
  1702. {
  1703. out.append(" <Stats>");
  1704. ForEachItemIn(i, stats)
  1705. stats.item(i).toXML(out);
  1706. out.append(" </Stats>\n");
  1707. }
  1708. SuperHashIteratorOf<CStatisticCollection> iter(children, false);
  1709. for (iter.first(); iter.isValid(); iter.next())
  1710. iter.query().toXML(out);
  1711. out.append("</Scope>\n");
  1712. return out;
  1713. }
  1714. static int compareCollection(IInterface * const * pl, IInterface * const *pr)
  1715. {
  1716. CStatisticCollection * l = static_cast<CStatisticCollection *>(static_cast<IStatisticCollection *>(*pl));
  1717. CStatisticCollection * r = static_cast<CStatisticCollection *>(static_cast<IStatisticCollection *>(*pr));
  1718. return l->queryScopeId().compare(r->queryScopeId());
  1719. }
  1720. //---------------------------------------------------------------------------------------------------------------------
  1721. void CollectionHashTable::onAdd(void *et)
  1722. {
  1723. }
  1724. void CollectionHashTable::onRemove(void *et)
  1725. {
  1726. CStatisticCollection * elem = reinterpret_cast<CStatisticCollection *>(et);
  1727. elem->Release();
  1728. }
  1729. unsigned CollectionHashTable::getHashFromElement(const void *et) const
  1730. {
  1731. const CStatisticCollection * elem = reinterpret_cast<const CStatisticCollection *>(et);
  1732. return elem->id.getHash();
  1733. }
  1734. unsigned CollectionHashTable::getHashFromFindParam(const void *fp) const
  1735. {
  1736. const StatsScopeId * search = reinterpret_cast<const StatsScopeId *>(fp);
  1737. return search->getHash();
  1738. }
  1739. const void * CollectionHashTable::getFindParam(const void *et) const
  1740. {
  1741. const CStatisticCollection * elem = reinterpret_cast<const CStatisticCollection *>(et);
  1742. return &elem->id;
  1743. }
  1744. bool CollectionHashTable::matchesFindParam(const void *et, const void *key, unsigned fphash) const
  1745. {
  1746. const CStatisticCollection * elem = reinterpret_cast<const CStatisticCollection *>(et);
  1747. const StatsScopeId * search = reinterpret_cast<const StatsScopeId *>(key);
  1748. return elem->id.matches(*search);
  1749. }
  1750. bool CollectionHashTable::matchesElement(const void *et, const void *searchET) const
  1751. {
  1752. const CStatisticCollection * elem = reinterpret_cast<const CStatisticCollection *>(et);
  1753. const CStatisticCollection * searchElem = reinterpret_cast<const CStatisticCollection *>(searchET);
  1754. return elem->id.matches(searchElem->id);
  1755. }
  1756. //---------------------------------------------------------------------------------------------------------------------
  1757. class CRootStatisticCollection : public CStatisticCollection
  1758. {
  1759. public:
  1760. CRootStatisticCollection(StatisticCreatorType _creatorType, const char * _creator, const StatsScopeId & _id)
  1761. : CStatisticCollection(NULL, _id), creatorType(_creatorType), creator(_creator)
  1762. {
  1763. whenCreated = getTimeStampNowValue();
  1764. }
  1765. CRootStatisticCollection(MemoryBuffer & in, unsigned version) : CStatisticCollection(NULL, in, version)
  1766. {
  1767. byte creatorTypeByte;
  1768. in.read(creatorTypeByte);
  1769. creatorType = (StatisticCreatorType)creatorTypeByte;
  1770. in.read(creator);
  1771. in.read(whenCreated);
  1772. }
  1773. virtual byte getCollectionType() const { return SCroot; }
  1774. virtual unsigned __int64 queryWhenCreated() const
  1775. {
  1776. return whenCreated;
  1777. }
  1778. virtual void serialize(MemoryBuffer & out) const
  1779. {
  1780. CStatisticCollection::serialize(out);
  1781. out.append((byte)creatorType);
  1782. out.append(creator);
  1783. out.append(whenCreated);
  1784. }
  1785. virtual void mergeInto(IStatisticGatherer & target) const override
  1786. {
  1787. // Similar to CStatisticCollection::mergeInfo but do not add the root scope.
  1788. ForEachItemIn(iStat, stats)
  1789. stats.item(iStat).mergeInto(target);
  1790. for (auto const & cur : children)
  1791. cur.mergeInto(target);
  1792. }
  1793. public:
  1794. StatisticCreatorType creatorType;
  1795. StringAttr creator;
  1796. unsigned __int64 whenCreated;
  1797. };
  1798. //---------------------------------------------------------------------------------------------------------------------
  1799. void serializeStatisticCollection(MemoryBuffer & out, IStatisticCollection * collection)
  1800. {
  1801. out.append(currentStatisticsVersion);
  1802. collection->serialize(out);
  1803. }
  1804. static CStatisticCollection * deserializeCollection(CStatisticCollection * parent, MemoryBuffer & in, unsigned version)
  1805. {
  1806. byte kind;
  1807. in.read(kind);
  1808. switch (kind)
  1809. {
  1810. case SCroot:
  1811. assertex(!parent);
  1812. return new CRootStatisticCollection(in, version);
  1813. case SCintermediate:
  1814. return new CStatisticCollection(parent, in, version);
  1815. default:
  1816. UNIMPLEMENTED;
  1817. }
  1818. }
  1819. IStatisticCollection * createStatisticCollection(MemoryBuffer & in)
  1820. {
  1821. unsigned version;
  1822. in.read(version);
  1823. return deserializeCollection(NULL, in, version);
  1824. }
  1825. //--------------------------------------------------------------------------------------------------------------------
  1826. class StatisticGatherer : implements CInterfaceOf<IStatisticGatherer>
  1827. {
  1828. public:
  1829. StatisticGatherer(CStatisticCollection * scope) : rootScope(scope)
  1830. {
  1831. scopes.append(*scope);
  1832. }
  1833. virtual void beginScope(const StatsScopeId & id) override
  1834. {
  1835. CStatisticCollection & tos = scopes.tos();
  1836. scopes.append(*tos.ensureSubScope(id, true));
  1837. }
  1838. virtual void beginActivityScope(unsigned id) override
  1839. {
  1840. StatsScopeId scopeId(SSTactivity, id);
  1841. CStatisticCollection & tos = scopes.tos();
  1842. scopes.append(*tos.ensureSubScope(scopeId, false));
  1843. }
  1844. virtual void beginSubGraphScope(unsigned id) override
  1845. {
  1846. StatsScopeId scopeId(SSTsubgraph, id);
  1847. CStatisticCollection & tos = scopes.tos();
  1848. scopes.append(*tos.ensureSubScope(scopeId, true));
  1849. }
  1850. virtual void beginEdgeScope(unsigned id, unsigned oid) override
  1851. {
  1852. StatsScopeId scopeId(SSTedge, id, oid);
  1853. CStatisticCollection & tos = scopes.tos();
  1854. scopes.append(*tos.ensureSubScope(scopeId, false));
  1855. }
  1856. virtual void beginChildGraphScope(unsigned id) override
  1857. {
  1858. StatsScopeId scopeId(SSTchildgraph, id);
  1859. CStatisticCollection & tos = scopes.tos();
  1860. scopes.append(*tos.ensureSubScope(scopeId, true));
  1861. }
  1862. virtual void beginChannelScope(unsigned id) override
  1863. {
  1864. StatsScopeId scopeId(SSTchannel, id);
  1865. CStatisticCollection & tos = scopes.tos();
  1866. scopes.append(*tos.ensureSubScope(scopeId, true));
  1867. }
  1868. virtual void endScope() override
  1869. {
  1870. scopes.pop();
  1871. }
  1872. virtual void addStatistic(StatisticKind kind, unsigned __int64 value) override
  1873. {
  1874. CStatisticCollection & tos = scopes.tos();
  1875. tos.addStatistic(kind, value);
  1876. }
  1877. virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction) override
  1878. {
  1879. CStatisticCollection & tos = scopes.tos();
  1880. tos.updateStatistic(kind, value, mergeAction);
  1881. }
  1882. virtual IStatisticCollection * getResult() override
  1883. {
  1884. return LINK(rootScope);
  1885. }
  1886. protected:
  1887. ICopyArrayOf<CStatisticCollection> scopes;
  1888. Linked<CStatisticCollection> rootScope;
  1889. };
  1890. extern IStatisticGatherer * createStatisticsGatherer(StatisticCreatorType creatorType, const char * creator, const StatsScopeId & rootScope)
  1891. {
  1892. //creator unused at the moment.
  1893. Owned<CStatisticCollection> rootCollection = new CRootStatisticCollection(creatorType, creator, rootScope);
  1894. return new StatisticGatherer(rootCollection);
  1895. }
  1896. //--------------------------------------------------------------------------------------------------------------------
  1897. extern IPropertyTree * selectTreeStat(IPropertyTree *node, const char *statName, const char *statType)
  1898. {
  1899. StringBuffer xpath;
  1900. xpath.appendf("att[@name='%s']", statName);
  1901. IPropertyTree *att = node->queryPropTree(xpath.str());
  1902. if (!att)
  1903. {
  1904. att = node->addPropTree("att", createPTree());
  1905. att->setProp("@name", statName);
  1906. att->setProp("@type", statType);
  1907. }
  1908. return att;
  1909. }
  1910. extern void putStatsTreeValue(IPropertyTree *node, const char *statName, const char *statType, unsigned __int64 val)
  1911. {
  1912. if (val)
  1913. selectTreeStat(node, statName, statType)->setPropInt64("@value", val);
  1914. }
  1915. class TreeNodeStatisticGatherer : public CInterfaceOf<IStatisticGatherer>
  1916. {
  1917. public:
  1918. TreeNodeStatisticGatherer(IPropertyTree & root) : node(&root) {}
  1919. virtual void beginScope(const StatsScopeId & id)
  1920. {
  1921. StringBuffer temp;
  1922. id.getScopeText(temp);
  1923. beginScope(temp.str());
  1924. }
  1925. virtual void beginSubGraphScope(unsigned id) { throwUnexpected(); }
  1926. virtual void beginChildGraphScope(unsigned id) { throwUnexpected(); }
  1927. virtual void beginActivityScope(unsigned id) { throwUnexpected(); }
  1928. virtual void beginEdgeScope(unsigned id, unsigned oid) { throwUnexpected(); }
  1929. virtual void beginChannelScope(unsigned id) { throwUnexpected(); }
  1930. virtual void endScope()
  1931. {
  1932. node = &stack.popGet();
  1933. }
  1934. virtual void addStatistic(StatisticKind kind, unsigned __int64 value)
  1935. {
  1936. putStatsTreeValue(node, queryStatisticName(kind), "sum", value);
  1937. }
  1938. virtual void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
  1939. {
  1940. if (value)
  1941. {
  1942. IPropertyTree * stat = selectTreeStat(node, queryStatisticName(kind), "sum");
  1943. unsigned __int64 newValue = mergeStatisticValue(stat->getPropInt64("@value"), value, mergeAction);
  1944. stat->setPropInt64("@value", newValue);
  1945. }
  1946. }
  1947. virtual IStatisticCollection * getResult() { throwUnexpected(); }
  1948. protected:
  1949. void beginScope(const char * id)
  1950. {
  1951. stack.append(*node);
  1952. StringBuffer xpath;
  1953. xpath.appendf("scope[@name='%s']", id);
  1954. IPropertyTree *att = node->queryPropTree(xpath.str());
  1955. if (!att)
  1956. {
  1957. att = node->addPropTree("scope", createPTree());
  1958. att->setProp("@name", id);
  1959. }
  1960. node = att;
  1961. }
  1962. protected:
  1963. IPropertyTree * node;
  1964. ICopyArrayOf<IPropertyTree> stack;
  1965. };
  1966. //--------------------------------------------------------------------------------------------------------------------
  1967. void CRuntimeStatistic::merge(unsigned __int64 otherValue, StatsMergeAction mergeAction)
  1968. {
  1969. switch (mergeAction)
  1970. {
  1971. case StatsMergeKeepNonZero:
  1972. if (otherValue && !value)
  1973. {
  1974. unsigned __int64 zero = 0;
  1975. value.compare_exchange_strong(zero, otherValue);
  1976. }
  1977. break;
  1978. case StatsMergeAppend:
  1979. case StatsMergeReplace:
  1980. value = otherValue;
  1981. break;
  1982. case StatsMergeSum:
  1983. if (otherValue)
  1984. addAtomic(otherValue);
  1985. break;
  1986. case StatsMergeMin:
  1987. value.store_min(otherValue);
  1988. break;
  1989. case StatsMergeMax:
  1990. value.store_max(otherValue);
  1991. break;
  1992. default:
  1993. #ifdef _DEBUG
  1994. throwUnexpected();
  1995. #else
  1996. value = otherValue;
  1997. #endif
  1998. }
  1999. }
  2000. //--------------------------------------------------------------------------------------------------------------------
  2001. CRuntimeStatisticCollection::~CRuntimeStatisticCollection()
  2002. {
  2003. delete [] values;
  2004. delete queryNested();
  2005. }
  2006. CNestedRuntimeStatisticMap * CRuntimeStatisticCollection::queryNested() const
  2007. {
  2008. return nested.load(std::memory_order_relaxed);
  2009. }
  2010. CNestedRuntimeStatisticMap * CRuntimeStatisticCollection::createNested() const
  2011. {
  2012. return new CNestedRuntimeStatisticMap;
  2013. }
  2014. CNestedRuntimeStatisticMap & CRuntimeStatisticCollection::ensureNested()
  2015. {
  2016. return *querySingleton(nested, nestlock, [this]{ return this->createNested(); });
  2017. }
  2018. CriticalSection CRuntimeStatisticCollection::nestlock;
  2019. unsigned __int64 CRuntimeStatisticCollection::getSerialStatisticValue(StatisticKind kind) const
  2020. {
  2021. unsigned __int64 value = getStatisticValue(kind);
  2022. StatisticKind rawKind= queryRawKind(kind);
  2023. if (kind == rawKind)
  2024. return value;
  2025. unsigned __int64 rawValue = getStatisticValue(rawKind);
  2026. return value + convertMeasure(rawKind, kind, rawValue);
  2027. }
  2028. void CRuntimeStatisticCollection::set(const CRuntimeStatisticCollection & other, unsigned node)
  2029. {
  2030. ForEachItemIn(i, other)
  2031. {
  2032. StatisticKind kind = other.getKind(i);
  2033. unsigned __int64 value = other.getStatisticValue(kind);
  2034. setStatistic(kind, value, node);
  2035. }
  2036. CNestedRuntimeStatisticMap *otherNested = other.queryNested();
  2037. if (otherNested)
  2038. {
  2039. ensureNested().set(*otherNested, node);
  2040. }
  2041. }
  2042. void CRuntimeStatisticCollection::merge(const CRuntimeStatisticCollection & other, unsigned node)
  2043. {
  2044. ForEachItemIn(i, other)
  2045. {
  2046. StatisticKind kind = other.getKind(i);
  2047. unsigned __int64 value = other.getStatisticValue(kind);
  2048. mergeStatistic(kind, value, node);
  2049. }
  2050. CNestedRuntimeStatisticMap *otherNested = other.queryNested();
  2051. if (otherNested)
  2052. {
  2053. ensureNested().merge(*otherNested, node);
  2054. }
  2055. }
  2056. void CRuntimeStatisticCollection::updateDelta(CRuntimeStatisticCollection & target, const CRuntimeStatisticCollection & source)
  2057. {
  2058. ForEachItemIn(i, source)
  2059. {
  2060. StatisticKind kind = source.getKind(i);
  2061. unsigned __int64 sourceValue = source.getStatisticValue(kind);
  2062. if (queryMergeMode(kind) == StatsMergeSum)
  2063. {
  2064. unsigned __int64 prevValue = getStatisticValue(kind);
  2065. if (sourceValue != prevValue)
  2066. {
  2067. target.mergeStatistic(kind, sourceValue - prevValue);
  2068. setStatistic(kind, sourceValue);
  2069. }
  2070. }
  2071. else
  2072. {
  2073. if (sourceValue)
  2074. target.mergeStatistic(kind, sourceValue);
  2075. }
  2076. }
  2077. CNestedRuntimeStatisticMap *sourceNested = source.queryNested();
  2078. if (sourceNested)
  2079. {
  2080. ensureNested().updateDelta(target.ensureNested(), *sourceNested);
  2081. }
  2082. }
  2083. void CRuntimeStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __int64 value)
  2084. {
  2085. CRuntimeStatistic * target = queryOptStatistic(kind);
  2086. if (target)
  2087. target->merge(value, queryMergeMode(kind));
  2088. }
  2089. void CRuntimeStatisticCollection::sumStatistic(StatisticKind kind, unsigned __int64 value)
  2090. {
  2091. queryStatistic(kind).sum(value);
  2092. }
  2093. void CRuntimeStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __int64 value, unsigned node)
  2094. {
  2095. mergeStatistic(kind, value);
  2096. }
  2097. void CRuntimeStatisticCollection::setStatistic(StatisticKind kind, unsigned __int64 value, unsigned node)
  2098. {
  2099. setStatistic(kind, value);
  2100. }
  2101. void CRuntimeStatisticCollection::reset()
  2102. {
  2103. unsigned num = mapping.numStatistics();
  2104. for (unsigned i = 0; i <= num; i++)
  2105. values[i].clear();
  2106. }
  2107. void CRuntimeStatisticCollection::reset(const StatisticsMapping & toClear)
  2108. {
  2109. unsigned num = toClear.numStatistics();
  2110. for (unsigned i = 0; i < num; i++)
  2111. queryStatistic(toClear.getKind(i)).clear();
  2112. }
  2113. CRuntimeStatisticCollection & CRuntimeStatisticCollection::registerNested(const StatsScopeId & scope, const StatisticsMapping & mapping)
  2114. {
  2115. return ensureNested().addNested(scope, mapping).queryStats();
  2116. }
  2117. void CRuntimeStatisticCollection::rollupStatistics(unsigned numTargets, IContextLogger * const * targets) const
  2118. {
  2119. ForEachItem(iStat)
  2120. {
  2121. unsigned __int64 value = values[iStat].getClear();
  2122. if (value)
  2123. {
  2124. StatisticKind kind = getKind(iStat);
  2125. for (unsigned iTarget = 0; iTarget < numTargets; iTarget++)
  2126. targets[iTarget]->noteStatistic(kind, value);
  2127. }
  2128. }
  2129. reportIgnoredStats();
  2130. }
  2131. void CRuntimeStatisticCollection::recordStatistics(IStatisticGatherer & target, bool clear) const
  2132. {
  2133. ForEachItem(i)
  2134. {
  2135. StatisticKind kind = getKind(i);
  2136. unsigned __int64 value = clear ? values[i].getClearAtomic() : values[i].get();
  2137. if (value || includeStatisticIfZero(kind))
  2138. {
  2139. StatisticKind serialKind= querySerializedKind(kind);
  2140. value = convertMeasure(kind, serialKind, value);
  2141. StatsMergeAction mergeAction = queryMergeMode(serialKind);
  2142. target.updateStatistic(serialKind, value, mergeAction);
  2143. }
  2144. }
  2145. reportIgnoredStats();
  2146. CNestedRuntimeStatisticMap *qn = queryNested();
  2147. if (qn)
  2148. qn->recordStatistics(target, clear);
  2149. }
  2150. void CRuntimeStatisticCollection::reportIgnoredStats() const
  2151. {
  2152. if (values[mapping.numStatistics()].getClear())
  2153. DBGLOG("Some statistics were added but thrown away");
  2154. }
  2155. StringBuffer & CRuntimeStatisticCollection::toXML(StringBuffer &str) const
  2156. {
  2157. ForEachItem(iStat)
  2158. {
  2159. unsigned __int64 value = values[iStat].get();
  2160. if (value)
  2161. {
  2162. StatisticKind kind = getKind(iStat);
  2163. const char * name = queryStatisticName(kind);
  2164. str.appendf("<%s>%" I64F "d</%s>", name, value, name);
  2165. }
  2166. }
  2167. CNestedRuntimeStatisticMap *qn = queryNested();
  2168. if (qn)
  2169. qn->toXML(str);
  2170. return str;
  2171. }
  2172. StringBuffer & CRuntimeStatisticCollection::toStr(StringBuffer &str) const
  2173. {
  2174. ForEachItem(iStat)
  2175. {
  2176. unsigned __int64 value = values[iStat].get();
  2177. if (value)
  2178. {
  2179. StatisticKind kind = getKind(iStat);
  2180. StatisticKind serialKind = querySerializedKind(kind);
  2181. if (kind != serialKind)
  2182. value = convertMeasure(kind, serialKind, value);
  2183. const char * name = queryStatisticName(serialKind);
  2184. str.append(' ').append(name).append("=");
  2185. formatStatistic(str, value, serialKind);
  2186. }
  2187. }
  2188. CNestedRuntimeStatisticMap *qn = queryNested();
  2189. if (qn)
  2190. qn->toStr(str);
  2191. return str;
  2192. }
  2193. void CRuntimeStatisticCollection::deserialize(MemoryBuffer& in)
  2194. {
  2195. unsigned numValid;
  2196. in.readPacked(numValid);
  2197. for (unsigned i=0; i < numValid; i++)
  2198. {
  2199. unsigned kindVal;
  2200. unsigned __int64 value;
  2201. in.readPacked(kindVal).readPacked(value);
  2202. StatisticKind kind = (StatisticKind)kindVal;
  2203. setStatistic(kind, value);
  2204. }
  2205. bool hasNested;
  2206. in.read(hasNested);
  2207. if (hasNested)
  2208. {
  2209. ensureNested().deserializeMerge(in);
  2210. }
  2211. }
  2212. void CRuntimeStatisticCollection::deserializeMerge(MemoryBuffer& in)
  2213. {
  2214. unsigned numValid;
  2215. in.readPacked(numValid);
  2216. for (unsigned i=0; i < numValid; i++)
  2217. {
  2218. unsigned kindVal;
  2219. unsigned __int64 value;
  2220. in.readPacked(kindVal).readPacked(value);
  2221. StatisticKind kind = (StatisticKind)kindVal;
  2222. StatsMergeAction mergeAction = queryMergeMode(kind);
  2223. mergeStatistic(kind, value, mergeAction);
  2224. }
  2225. bool hasNested;
  2226. in.read(hasNested);
  2227. if (hasNested)
  2228. {
  2229. ensureNested().deserializeMerge(in);
  2230. }
  2231. }
  2232. bool CRuntimeStatisticCollection::serialize(MemoryBuffer& out) const
  2233. {
  2234. unsigned numValid = 0;
  2235. ForEachItem(i1)
  2236. {
  2237. if (values[i1].get())
  2238. numValid++;
  2239. }
  2240. out.appendPacked(numValid);
  2241. ForEachItem(i2)
  2242. {
  2243. unsigned __int64 value = values[i2].get();
  2244. if (value)
  2245. {
  2246. StatisticKind kind = mapping.getKind(i2);
  2247. StatisticKind serialKind= querySerializedKind(kind);
  2248. if (kind != serialKind)
  2249. value = convertMeasure(kind, serialKind, value);
  2250. out.appendPacked((unsigned)serialKind);
  2251. out.appendPacked(value);
  2252. }
  2253. }
  2254. bool nonEmpty = (numValid != 0);
  2255. CNestedRuntimeStatisticMap *qn = queryNested();
  2256. out.append(qn != nullptr);
  2257. if (qn)
  2258. {
  2259. if (qn->serialize(out))
  2260. nonEmpty = true;
  2261. }
  2262. return nonEmpty;
  2263. }
  2264. //---------------------------------------------------------------------------------------------------------------------
  2265. void CRuntimeSummaryStatisticCollection::DerivedStats::mergeStatistic(unsigned __int64 value, unsigned node)
  2266. {
  2267. if (count == 0)
  2268. {
  2269. min = value;
  2270. max = value;
  2271. minNode = node;
  2272. maxNode = node;
  2273. }
  2274. else
  2275. {
  2276. if (value < min)
  2277. {
  2278. min = value;
  2279. minNode = node;
  2280. }
  2281. if (value > max)
  2282. {
  2283. max = value;
  2284. maxNode = node;
  2285. }
  2286. }
  2287. count++;
  2288. sum += value;
  2289. double dvalue = (double)value;
  2290. sumSquares += dvalue * dvalue;
  2291. }
  2292. void CRuntimeSummaryStatisticCollection::DerivedStats::setStatistic(unsigned __int64 value, unsigned node)
  2293. {
  2294. if (count == 0)
  2295. {
  2296. min = value;
  2297. max = value;
  2298. minNode = node;
  2299. maxNode = node;
  2300. }
  2301. else
  2302. {
  2303. if (value < min)
  2304. {
  2305. min = value;
  2306. minNode = node;
  2307. }
  2308. if (value > max)
  2309. {
  2310. max = value;
  2311. maxNode = node;
  2312. }
  2313. }
  2314. count++;
  2315. sum = value;
  2316. double dvalue = (double)value;
  2317. sumSquares = dvalue * dvalue;
  2318. }
  2319. double CRuntimeSummaryStatisticCollection::DerivedStats::queryStdDevInfo(unsigned __int64 &_min, unsigned __int64 &_max, unsigned &_minNode, unsigned &_maxNode) const
  2320. {
  2321. _min = min;
  2322. _max = max;
  2323. _minNode = minNode;
  2324. _maxNode = maxNode;
  2325. double mean = sum / count;
  2326. double variance = (sumSquares - sum * mean) / count;
  2327. return sqrt(variance);
  2328. }
  2329. CRuntimeSummaryStatisticCollection::CRuntimeSummaryStatisticCollection(const StatisticsMapping & _mapping) : CRuntimeStatisticCollection(_mapping)
  2330. {
  2331. derived = new DerivedStats[ordinality()+1];
  2332. }
  2333. CRuntimeSummaryStatisticCollection::~CRuntimeSummaryStatisticCollection()
  2334. {
  2335. delete[] derived;
  2336. }
  2337. CNestedRuntimeStatisticMap * CRuntimeSummaryStatisticCollection::createNested() const
  2338. {
  2339. return new CNestedSummaryRuntimeStatisticMap;
  2340. }
  2341. void CRuntimeSummaryStatisticCollection::mergeStatistic(StatisticKind kind, unsigned __int64 value, unsigned node)
  2342. {
  2343. CRuntimeStatisticCollection::mergeStatistic(kind, value);
  2344. unsigned index = queryMapping().getIndex(kind);
  2345. derived[index].mergeStatistic(value, node);
  2346. }
  2347. void CRuntimeSummaryStatisticCollection::setStatistic(StatisticKind kind, unsigned __int64 value, unsigned node)
  2348. {
  2349. CRuntimeStatisticCollection::setStatistic(kind, value);
  2350. unsigned index = queryMapping().getIndex(kind);
  2351. derived[index].setStatistic(value, node);
  2352. }
  2353. double CRuntimeSummaryStatisticCollection::queryStdDevInfo(StatisticKind kind, unsigned __int64 &_min, unsigned __int64 &_max, unsigned &_minNode, unsigned &_maxNode) const
  2354. {
  2355. return derived[queryMapping().getIndex(kind)].queryStdDevInfo(_min, _max, _minNode, _maxNode);
  2356. }
  2357. static bool skewHasMeaning(StatisticKind kind)
  2358. {
  2359. //Check that skew makes any sense for the type of measurement
  2360. switch (queryMeasure(kind))
  2361. {
  2362. case SMeasureTimeNs:
  2363. case SMeasureCount:
  2364. case SMeasureSize:
  2365. return true;
  2366. default:
  2367. return false;
  2368. }
  2369. }
  2370. static bool isSignificantRange(StatisticKind kind, unsigned __int64 range, unsigned __int64 mean)
  2371. {
  2372. //Ignore tiny differences (often occur with counts of single rows on 1 slave node)
  2373. unsigned insignificantDiff = 1;
  2374. switch (queryMeasure(kind))
  2375. {
  2376. case SMeasureTimestampUs:
  2377. insignificantDiff = 1000; // Ignore 1ms timestamp difference between nodes
  2378. break;
  2379. case SMeasureTimeNs:
  2380. insignificantDiff = 1000; // Ignore 1us timing difference between nodes
  2381. break;
  2382. case SMeasureSize:
  2383. insignificantDiff = 1024;
  2384. break;
  2385. }
  2386. if (range <= insignificantDiff)
  2387. return false;
  2388. if (queryMergeMode(kind) == StatsMergeSum)
  2389. {
  2390. //if the range is < 0.01% of the mean, then it is unlikely to be interesting
  2391. if (range * 10000 < mean)
  2392. return false;
  2393. }
  2394. return true;
  2395. }
  2396. static bool isWorthReportingMergedValue(StatisticKind kind)
  2397. {
  2398. switch (queryMergeMode(kind))
  2399. {
  2400. //Does the merged value have a meaning?
  2401. case StatsMergeSum:
  2402. case StatsMergeMin:
  2403. case StatsMergeMax:
  2404. break;
  2405. default:
  2406. return false;
  2407. }
  2408. switch (queryMeasure(kind))
  2409. {
  2410. case SMeasureTimeNs:
  2411. //Not generally worth reporting the total time across all slaves
  2412. return false;
  2413. }
  2414. return true;
  2415. }
  2416. void CRuntimeSummaryStatisticCollection::recordStatistics(IStatisticGatherer & target, bool clear) const
  2417. {
  2418. for (unsigned i = 0; i < ordinality(); i++)
  2419. {
  2420. DerivedStats & cur = derived[i];
  2421. StatisticKind kind = getKind(i);
  2422. StatisticKind serialKind = querySerializedKind(kind);
  2423. if (cur.count)
  2424. {
  2425. //Thor should always publish the average value for a stat, and the merged value if it makes sense.
  2426. //So that it is easy to analyse graphs independent of the number of slave nodes it is executed on.
  2427. unsigned __int64 mergedValue = convertMeasure(kind, serialKind, clear ? values[i].getClearAtomic() : values[i].get());
  2428. if (isWorthReportingMergedValue(serialKind))
  2429. {
  2430. if (mergedValue || includeStatisticIfZero(serialKind))
  2431. target.addStatistic(serialKind, mergedValue);
  2432. }
  2433. unsigned __int64 minValue = convertMeasure(kind, serialKind, cur.min);
  2434. unsigned __int64 maxValue = convertMeasure(kind, serialKind, cur.max);
  2435. if (minValue != maxValue)
  2436. {
  2437. //Avoid rounding errors summing values as doubles - if they were also summed as integers. Probably overkill!
  2438. //There may still be noticeable rounding errors with timestamps... revisit if it is an issue with any measurement
  2439. double sum = (queryMergeMode(kind) == StatsMergeSum) ? (double)mergedValue : convertSumMeasure(kind, serialKind, cur.sum);
  2440. double mean = (double)(sum / cur.count);
  2441. unsigned __int64 range = maxValue - minValue;
  2442. target.addStatistic((StatisticKind)(serialKind|StAvgX), (unsigned __int64)mean);
  2443. target.addStatistic((StatisticKind)(serialKind|StMinX), minValue);
  2444. target.addStatistic((StatisticKind)(serialKind|StMaxX), maxValue);
  2445. //Exclude delta and std dev if a single node was the only one that provided a value
  2446. if ((minValue != 0) || (maxValue != mergedValue))
  2447. {
  2448. //The delta/std dev may have a different unit from the original values e.g., timestamps->times, so needs scaling
  2449. unsigned __int64 scaledRange = convertMeasure(serialKind, serialKind|StDeltaX, range);
  2450. target.addStatistic(serialKind|StDeltaX, scaledRange);
  2451. if (skewHasMeaning(serialKind))
  2452. {
  2453. //Sum of squares needs to be translated twice
  2454. double sumSquares = convertSquareMeasure(kind, serialKind, cur.sumSquares);
  2455. double variance = (sumSquares - sum * mean) / cur.count;
  2456. double stdDev = sqrt(variance);
  2457. unsigned __int64 scaledStdDev = convertMeasure(serialKind, serialKind|StStdDevX, stdDev);
  2458. target.addStatistic(serialKind|StStdDevX, scaledStdDev);
  2459. }
  2460. }
  2461. //First test is redundant - but protects against minValue != maxValue test above changing.
  2462. if ((cur.minNode != cur.maxNode) && isSignificantRange(serialKind, range, mean))
  2463. {
  2464. target.addStatistic((StatisticKind)(serialKind|StNodeMin), cur.minNode);
  2465. target.addStatistic((StatisticKind)(serialKind|StNodeMax), cur.maxNode);
  2466. if (skewHasMeaning(serialKind))
  2467. {
  2468. double maxSkew = (10000.0 * ((maxValue-mean)/mean));
  2469. double minSkew = (10000.0 * ((mean-minValue)/mean));
  2470. target.addStatistic((StatisticKind)(serialKind|StSkewMin), (unsigned __int64)minSkew);
  2471. target.addStatistic((StatisticKind)(serialKind|StSkewMax), (unsigned __int64)maxSkew);
  2472. }
  2473. }
  2474. }
  2475. else
  2476. {
  2477. if (minValue || includeStatisticIfZero(serialKind))
  2478. target.addStatistic((StatisticKind)(serialKind|StAvgX), minValue);
  2479. }
  2480. }
  2481. else
  2482. {
  2483. //No results received from any of the slave yet... so do not report any stats
  2484. }
  2485. }
  2486. reportIgnoredStats();
  2487. CNestedRuntimeStatisticMap *qn = queryNested();
  2488. if (qn)
  2489. qn->recordStatistics(target, clear);
  2490. }
  2491. bool CRuntimeSummaryStatisticCollection::serialize(MemoryBuffer & out) const
  2492. {
  2493. UNIMPLEMENTED; // NB: Need to convert sum squares twice.
  2494. }
  2495. void CRuntimeSummaryStatisticCollection::deserialize(MemoryBuffer & in)
  2496. {
  2497. UNIMPLEMENTED;
  2498. }
  2499. void CRuntimeSummaryStatisticCollection::deserializeMerge(MemoryBuffer& in)
  2500. {
  2501. UNIMPLEMENTED;
  2502. }
  2503. //---------------------------------------------------
  2504. bool CNestedRuntimeStatisticCollection::matches(const StatsScopeId & otherScope) const
  2505. {
  2506. return scope.matches(otherScope);
  2507. }
  2508. //NOTE: When deserializing, the scope is deserialized by the caller, and the correct target selected
  2509. //which is why there is no corresponding deserialize() ofthe scope at this point
  2510. void CNestedRuntimeStatisticCollection::deserialize(MemoryBuffer & in)
  2511. {
  2512. stats->deserialize(in);
  2513. }
  2514. void CNestedRuntimeStatisticCollection::deserializeMerge(MemoryBuffer& in)
  2515. {
  2516. stats->deserializeMerge(in);
  2517. }
  2518. void CNestedRuntimeStatisticCollection::merge(const CNestedRuntimeStatisticCollection & other, unsigned node)
  2519. {
  2520. stats->merge(other.queryStats(), node);
  2521. }
  2522. void CNestedRuntimeStatisticCollection::set(const CNestedRuntimeStatisticCollection & other, unsigned node)
  2523. {
  2524. stats->set(other.queryStats(), node);
  2525. }
  2526. bool CNestedRuntimeStatisticCollection::serialize(MemoryBuffer& out) const
  2527. {
  2528. scope.serialize(out);
  2529. return stats->serialize(out);
  2530. }
  2531. void CNestedRuntimeStatisticCollection::recordStatistics(IStatisticGatherer & target, bool clear) const
  2532. {
  2533. target.beginScope(scope);
  2534. stats->recordStatistics(target, clear);
  2535. target.endScope();
  2536. }
  2537. StringBuffer & CNestedRuntimeStatisticCollection::toStr(StringBuffer &str) const
  2538. {
  2539. str.append(' ');
  2540. scope.getScopeText(str).append("={");
  2541. stats->toStr(str);
  2542. return str.append(" }");
  2543. }
  2544. StringBuffer & CNestedRuntimeStatisticCollection::toXML(StringBuffer &str) const
  2545. {
  2546. str.append("<Scope id=\"");
  2547. scope.getScopeText(str).append("\">");
  2548. stats->toXML(str);
  2549. return str.append("</Scope>");
  2550. }
  2551. void CNestedRuntimeStatisticCollection::updateDelta(CNestedRuntimeStatisticCollection & target, const CNestedRuntimeStatisticCollection & source)
  2552. {
  2553. stats->updateDelta(*target.stats, *source.stats);
  2554. }
  2555. //---------------------------------------------------
  2556. CNestedRuntimeStatisticCollection & CNestedRuntimeStatisticMap::addNested(const StatsScopeId & scope, const StatisticsMapping & mapping)
  2557. {
  2558. unsigned mapSize;
  2559. unsigned entry;
  2560. {
  2561. ReadLockBlock b(lock);
  2562. mapSize = map.length();
  2563. for (entry = 0; entry < mapSize; entry++)
  2564. {
  2565. CNestedRuntimeStatisticCollection & cur = map.item(entry);
  2566. if (cur.matches(scope))
  2567. return cur;
  2568. }
  2569. }
  2570. {
  2571. WriteLockBlock b(lock);
  2572. // Check no-one added anything between the read and write locks
  2573. mapSize = map.length();
  2574. for (; entry < mapSize; entry++)
  2575. {
  2576. CNestedRuntimeStatisticCollection & cur = map.item(entry);
  2577. if (cur.matches(scope))
  2578. return cur;
  2579. }
  2580. CNestedRuntimeStatisticCollection * stats = new CNestedRuntimeStatisticCollection(scope, createStats(mapping));
  2581. map.append(*stats);
  2582. return *stats;
  2583. }
  2584. }
  2585. void CNestedRuntimeStatisticMap::deserialize(MemoryBuffer& in)
  2586. {
  2587. unsigned numItems;
  2588. in.readPacked(numItems);
  2589. for (unsigned i=0; i < numItems; i++)
  2590. {
  2591. StatsScopeId scope;
  2592. scope.deserialize(in, currentStatisticsVersion);
  2593. //Use allStatistics as the default mapping if it hasn't already been added.
  2594. CNestedRuntimeStatisticCollection & child = addNested(scope, allStatistics);
  2595. child.deserialize(in);
  2596. }
  2597. }
  2598. void CNestedRuntimeStatisticMap::deserializeMerge(MemoryBuffer& in)
  2599. {
  2600. unsigned numItems;
  2601. in.readPacked(numItems);
  2602. for (unsigned i=0; i < numItems; i++)
  2603. {
  2604. StatsScopeId scope;
  2605. scope.deserialize(in, currentStatisticsVersion);
  2606. //Use allStatistics as the default mapping if it hasn't already been added.
  2607. CNestedRuntimeStatisticCollection & child = addNested(scope, allStatistics);
  2608. child.deserializeMerge(in);
  2609. }
  2610. }
  2611. void CNestedRuntimeStatisticMap::merge(const CNestedRuntimeStatisticMap & other, unsigned node)
  2612. {
  2613. ReadLockBlock b(other.lock);
  2614. ForEachItemIn(i, other.map)
  2615. {
  2616. CNestedRuntimeStatisticCollection & cur = other.map.item(i);
  2617. CNestedRuntimeStatisticCollection & target = addNested(cur.scope, cur.queryMapping());
  2618. target.merge(cur, node);
  2619. }
  2620. }
  2621. void CNestedRuntimeStatisticMap::set(const CNestedRuntimeStatisticMap & other, unsigned node)
  2622. {
  2623. ReadLockBlock b(other.lock);
  2624. ForEachItemIn(i, other.map)
  2625. {
  2626. CNestedRuntimeStatisticCollection & cur = other.map.item(i);
  2627. CNestedRuntimeStatisticCollection & target = addNested(cur.scope, cur.queryMapping());
  2628. target.set(cur, node);
  2629. }
  2630. }
  2631. void CNestedRuntimeStatisticMap::updateDelta(CNestedRuntimeStatisticMap & target, const CNestedRuntimeStatisticMap & source)
  2632. {
  2633. ReadLockBlock b(source.lock);
  2634. ForEachItemIn(i, source.map)
  2635. {
  2636. CNestedRuntimeStatisticCollection & curSource = source.map.item(i);
  2637. CNestedRuntimeStatisticCollection & curTarget = target.addNested(curSource.scope, curSource.queryMapping());
  2638. CNestedRuntimeStatisticCollection & curDelta = addNested(curSource.scope, curSource.queryMapping());
  2639. curDelta.updateDelta(curTarget, curSource);
  2640. }
  2641. }
  2642. bool CNestedRuntimeStatisticMap::serialize(MemoryBuffer& out) const
  2643. {
  2644. ReadLockBlock b(lock);
  2645. out.appendPacked(map.ordinality());
  2646. bool nonEmpty = false;
  2647. ForEachItemIn(i, map)
  2648. {
  2649. if (map.item(i).serialize(out))
  2650. nonEmpty = true;
  2651. }
  2652. return nonEmpty;
  2653. }
  2654. void CNestedRuntimeStatisticMap::recordStatistics(IStatisticGatherer & target, bool clear) const
  2655. {
  2656. ReadLockBlock b(lock);
  2657. ForEachItemIn(i, map)
  2658. map.item(i).recordStatistics(target, clear);
  2659. }
  2660. StringBuffer & CNestedRuntimeStatisticMap::toStr(StringBuffer &str) const
  2661. {
  2662. ReadLockBlock b(lock);
  2663. ForEachItemIn(i, map)
  2664. map.item(i).toStr(str);
  2665. return str;
  2666. }
  2667. StringBuffer & CNestedRuntimeStatisticMap::toXML(StringBuffer &str) const
  2668. {
  2669. ReadLockBlock b(lock);
  2670. ForEachItemIn(i, map)
  2671. map.item(i).toXML(str);
  2672. return str;
  2673. }
  2674. CRuntimeStatisticCollection * CNestedRuntimeStatisticMap::createStats(const StatisticsMapping & mapping)
  2675. {
  2676. return new CRuntimeStatisticCollection(mapping);
  2677. }
  2678. CRuntimeStatisticCollection * CNestedSummaryRuntimeStatisticMap::createStats(const StatisticsMapping & mapping)
  2679. {
  2680. return new CRuntimeSummaryStatisticCollection(mapping);
  2681. }
  2682. //---------------------------------------------------
  2683. void StatsAggregation::noteValue(stat_type value)
  2684. {
  2685. if (count == 0)
  2686. {
  2687. minValue = value;
  2688. maxValue = value;
  2689. }
  2690. else
  2691. {
  2692. if (value < minValue)
  2693. minValue = value;
  2694. else if (value > maxValue)
  2695. maxValue = value;
  2696. }
  2697. count++;
  2698. sumValue += value;
  2699. }
  2700. stat_type StatsAggregation::getAve() const
  2701. {
  2702. return (sumValue / count);
  2703. }
  2704. //---------------------------------------------------
  2705. ScopeCompare compareScopes(const char * scope, const char * key)
  2706. {
  2707. byte left = *scope;
  2708. byte right = *key;
  2709. //Check for root scope "" compared with anything
  2710. if (!left)
  2711. {
  2712. if (!right)
  2713. return SCequal;
  2714. return SCparent;
  2715. }
  2716. else if (!right)
  2717. {
  2718. return SCchild;
  2719. }
  2720. bool hadCommonScope = false;
  2721. for (;;)
  2722. {
  2723. if (left != right)
  2724. {
  2725. //FUTURE: Extend this function to support skipping numbers to allow wildcard matching
  2726. if (!left)
  2727. {
  2728. if (right == ':')
  2729. return SCparent; // scope is a parent (prefix) of the key
  2730. }
  2731. if (!right)
  2732. {
  2733. if (left == ':')
  2734. return SCchild; // scope is a child (superset) of the key
  2735. }
  2736. return hadCommonScope ? SCrelated : SCunrelated;
  2737. }
  2738. if (!left)
  2739. return SCequal;
  2740. if (left == ':')
  2741. hadCommonScope = true;
  2742. left = *++scope;
  2743. right =*++key;
  2744. }
  2745. }
  2746. ScopeFilter::ScopeFilter(const char * scopeList)
  2747. {
  2748. //MORE: This currently expands a list of scopes - it should probably be improved
  2749. scopes.appendList(scopeList, ",");
  2750. }
  2751. void ScopeFilter::addScope(const char * scope)
  2752. {
  2753. if (!scope)
  2754. return;
  2755. if (streq(scope, "*"))
  2756. {
  2757. scopes.kill();
  2758. minDepth = 0;
  2759. maxDepth = UINT_MAX;
  2760. return;
  2761. }
  2762. if (ids)
  2763. throw makeStringExceptionV(0, "Cannot filter by id and scope in the same request");
  2764. unsigned depth = queryScopeDepth(scope);
  2765. if ((scopes.ordinality() == 0) || (depth < minDepth))
  2766. minDepth = depth;
  2767. if ((scopes.ordinality() == 0) || (depth > maxDepth))
  2768. maxDepth = depth;
  2769. scopes.append(scope);
  2770. }
  2771. void ScopeFilter::addScopes(const char * scope)
  2772. {
  2773. StringArray list;
  2774. list.appendList(scope, ",");
  2775. ForEachItemIn(i, list)
  2776. addScope(list.item(i));
  2777. }
  2778. void ScopeFilter::addScopeType(StatisticScopeType scopeType)
  2779. {
  2780. if (scopeType == SSTall)
  2781. return;
  2782. scopeTypes.append(scopeType);
  2783. }
  2784. void ScopeFilter::addId(const char * id)
  2785. {
  2786. if (scopes)
  2787. throw makeStringExceptionV(0, "Cannot filter by id and scope in the same request");
  2788. ids.append(id);
  2789. }
  2790. void ScopeFilter::setDepth(unsigned low, unsigned high)
  2791. {
  2792. if (low > high)
  2793. throw makeStringExceptionV(0, "Depth parameters in wrong order %u..%u", low, high);
  2794. minDepth = low;
  2795. maxDepth = high;
  2796. }
  2797. void ScopeFilter::intersectDepth(const unsigned low, const unsigned high)
  2798. {
  2799. if (low > high)
  2800. throw makeStringExceptionV(0, "Depth parameters in wrong order %u..%u", low, high);
  2801. if (minDepth < low)
  2802. minDepth = low;
  2803. if (maxDepth > high)
  2804. maxDepth = high;
  2805. }
  2806. ScopeCompare ScopeFilter::compare(const char * scope) const
  2807. {
  2808. ScopeCompare result = SCunknown;
  2809. if (scopes)
  2810. {
  2811. //If scopes have been provided, then we are searching for an exact match against that scope
  2812. ForEachItemIn(i, scopes)
  2813. result |= compareScopes(scope, scopes.item(i));
  2814. }
  2815. else
  2816. {
  2817. //How does the depth of the scope compare with the range we are expecting?
  2818. unsigned depth = queryScopeDepth(scope);
  2819. if (depth < minDepth)
  2820. return SCparent;
  2821. if (depth > maxDepth)
  2822. return SCchild;
  2823. //Assume it is a match until proven otherwise
  2824. result |= SCequal;
  2825. // Could be the child of a match
  2826. if (depth > minDepth)
  2827. result |= SCchild;
  2828. //Could be the parent of a match
  2829. if (depth < maxDepth)
  2830. result |= SCparent;
  2831. //Check if the type of the current object matches the type
  2832. const char * tail = queryScopeTail(scope);
  2833. if (scopeTypes.ordinality())
  2834. {
  2835. StatsScopeId id(tail);
  2836. if (!scopeTypes.contains(id.queryScopeType()))
  2837. result &= ~SCequal;
  2838. }
  2839. if (ids)
  2840. {
  2841. if (!ids.contains(tail))
  2842. result &= ~SCequal;
  2843. }
  2844. }
  2845. if (!(result & SCequal))
  2846. return result;
  2847. //Have a match - now check that the attributes match as required
  2848. //MORE:
  2849. if (false)
  2850. {
  2851. result &= ~SCequal;
  2852. }
  2853. return result;
  2854. }
  2855. bool ScopeFilter::canAlwaysPreFilter() const
  2856. {
  2857. //If the only filter being applied is a restriction on the minimum depth, then you can always apply it as a pre-filter
  2858. return (!ids && !scopeTypes && !scopes && maxDepth == UINT_MAX);
  2859. }
  2860. int ScopeFilter::compareDepth(unsigned depth) const
  2861. {
  2862. if (depth < minDepth)
  2863. return -1;
  2864. if (depth > maxDepth)
  2865. return +1;
  2866. return 0;
  2867. }
  2868. StringBuffer & ScopeFilter::describe(StringBuffer & out) const
  2869. {
  2870. if ((minDepth != 0) || (maxDepth != UINT_MAX))
  2871. {
  2872. if (minDepth == maxDepth)
  2873. out.appendf(",depth(%u)", minDepth);
  2874. else
  2875. out.appendf(",depth(%u,%u)", minDepth, maxDepth);
  2876. }
  2877. if (scopeTypes)
  2878. {
  2879. out.append(",stype[");
  2880. ForEachItemIn(i, scopeTypes)
  2881. {
  2882. if (i)
  2883. out.append(",");
  2884. out.append(queryScopeTypeName((StatisticScopeType)scopeTypes.item(i)));
  2885. }
  2886. out.append("]");
  2887. }
  2888. if (scopes)
  2889. {
  2890. out.append(",scope[");
  2891. ForEachItemIn(i, scopes)
  2892. {
  2893. if (i)
  2894. out.append(",");
  2895. out.append(scopes.item(i));
  2896. }
  2897. out.append("]");
  2898. }
  2899. if (ids)
  2900. {
  2901. out.append(",id[");
  2902. ForEachItemIn(i, ids)
  2903. {
  2904. if (i)
  2905. out.append(",");
  2906. out.append(ids.item(i));
  2907. }
  2908. out.append("]");
  2909. }
  2910. return out;
  2911. }
  2912. void ScopeFilter::finishedFilter()
  2913. {
  2914. //If scopeTypes are provided, then this code ensure that any scopes and ids match them
  2915. //but that would have little benefit, and would cause complications if the only id was removed
  2916. //Some scope types can only exist at a single level.
  2917. if (scopeTypes.ordinality() == 1)
  2918. {
  2919. switch (scopeTypes.item(0))
  2920. {
  2921. case SSTglobal:
  2922. intersectDepth(0, 0);
  2923. break;
  2924. case SSTworkflow:
  2925. intersectDepth(1, 1);
  2926. break;
  2927. case SSTgraph:
  2928. //This should really be intersectDepth(2,2) but workunits prior to 7.4 did not have graph ids prefixed by the wfid
  2929. //Remove once 7.2 is a distant memory (see HPCC-22887)
  2930. intersectDepth(1, 2);
  2931. break;
  2932. case SSTsubgraph:
  2933. intersectDepth(3, UINT_MAX);
  2934. break;
  2935. case SSTactivity:
  2936. intersectDepth(4, UINT_MAX);
  2937. break;
  2938. }
  2939. }
  2940. }
  2941. bool ScopeFilter::hasSingleMatch() const
  2942. {
  2943. return scopes.ordinality() == 1 || ids.ordinality() == 1;
  2944. }
  2945. bool ScopeFilter::matchOnly(StatisticScopeType scopeType) const
  2946. {
  2947. if ((scopeTypes.ordinality() == 1) && (scopeTypes.item(0) == scopeType))
  2948. return true;
  2949. //Check the types of the scopes that are being searched
  2950. if (scopes.ordinality())
  2951. {
  2952. ForEachItemIn(i, scopes)
  2953. {
  2954. const char * scopeId = queryScopeTail(scopes.item(i));
  2955. StatsScopeId id(scopeId);
  2956. if (id.queryScopeType() != scopeType)
  2957. return false;
  2958. }
  2959. return true;
  2960. }
  2961. if (ids.ordinality())
  2962. {
  2963. ForEachItemIn(i, ids)
  2964. {
  2965. const char * scopeId = ids.item(i);
  2966. StatsScopeId id(scopeId);
  2967. if (id.queryScopeType() != scopeType)
  2968. return false;
  2969. }
  2970. return true;
  2971. }
  2972. return false;
  2973. }
  2974. //---------------------------------------------------
  2975. bool ScopedItemFilter::matchDepth(unsigned low, unsigned high) const
  2976. {
  2977. if (maxDepth && low && maxDepth < low)
  2978. return false;
  2979. if (minDepth && high && minDepth > high)
  2980. return false;
  2981. return true;
  2982. }
  2983. bool ScopedItemFilter::match(const char * search) const
  2984. {
  2985. if (search)
  2986. {
  2987. if (value)
  2988. {
  2989. if (hasWildcard)
  2990. {
  2991. //MORE: If wildcarding ends up being used a lot then this should be replaced with something that creates a DFA
  2992. if (!WildMatch(search, value, false))
  2993. return false;
  2994. }
  2995. else
  2996. {
  2997. return streq(search, value);
  2998. }
  2999. }
  3000. if (minDepth || maxDepth)
  3001. {
  3002. unsigned searchDepth = queryScopeDepth(search);
  3003. if (searchDepth < minDepth)
  3004. return false;
  3005. if (maxDepth && searchDepth > maxDepth)
  3006. return false;
  3007. }
  3008. }
  3009. return true;
  3010. }
  3011. bool ScopedItemFilter::recurseChildScopes(const char * curScope) const
  3012. {
  3013. if (maxDepth == 0 || !curScope)
  3014. return true;
  3015. if (queryScopeDepth(curScope) >= maxDepth)
  3016. return false;
  3017. return true;
  3018. }
  3019. void ScopedItemFilter::set(const char * _value)
  3020. {
  3021. if (_value && !streq(_value, "*") )
  3022. {
  3023. value.set(_value);
  3024. minDepth = queryScopeDepth(_value);
  3025. if (!strchr(_value, '*'))
  3026. {
  3027. maxDepth = minDepth;
  3028. hasWildcard = strchr(_value, '?') != NULL;
  3029. }
  3030. else
  3031. hasWildcard = true;
  3032. }
  3033. else
  3034. value.clear();
  3035. }
  3036. void ScopedItemFilter::setDepth(unsigned _depth)
  3037. {
  3038. minDepth = _depth;
  3039. maxDepth = _depth;
  3040. }
  3041. void ScopedItemFilter::setDepth(unsigned _minDepth, unsigned _maxDepth)
  3042. {
  3043. minDepth = _minDepth;
  3044. maxDepth = _maxDepth;
  3045. }
  3046. StatisticsFilter::StatisticsFilter()
  3047. {
  3048. init();
  3049. }
  3050. StatisticsFilter::StatisticsFilter(const char * filter)
  3051. {
  3052. init();
  3053. setFilter(filter);
  3054. }
  3055. StatisticsFilter::StatisticsFilter(StatisticCreatorType _creatorType, StatisticScopeType _scopeType, StatisticMeasure _measure, StatisticKind _kind)
  3056. {
  3057. init();
  3058. creatorType = _creatorType;
  3059. scopeType = _scopeType;
  3060. measure = _measure;
  3061. kind = _kind;
  3062. }
  3063. StatisticsFilter::StatisticsFilter(const char * _creatorType, const char * _scopeType, const char * _kind)
  3064. {
  3065. init();
  3066. set(_creatorType, _scopeType, _kind);
  3067. }
  3068. StatisticsFilter::StatisticsFilter(const char * _creatorTypeText, const char * _creator, const char * _scopeTypeText, const char * _scope, const char * _measureText, const char * _kindText)
  3069. {
  3070. init();
  3071. set(_creatorTypeText, _creator, _scopeTypeText, _scope, _measureText, _kindText);
  3072. }
  3073. StatisticsFilter::StatisticsFilter(StatisticCreatorType _creatorType, const char * _creator, StatisticScopeType _scopeType, const char * _scope, StatisticMeasure _measure, StatisticKind _kind)
  3074. {
  3075. init();
  3076. creatorType = _creatorType;
  3077. setCreator(_creator);
  3078. scopeType = _scopeType;
  3079. setScope(_scope);
  3080. measure = _measure;
  3081. kind = _kind;
  3082. }
  3083. void StatisticsFilter::init()
  3084. {
  3085. creatorType = SCTall;
  3086. scopeType = SSTall;
  3087. measure = SMeasureAll;
  3088. kind = StKindAll;
  3089. }
  3090. bool StatisticsFilter::matches(StatisticCreatorType curCreatorType, const char * curCreator, StatisticScopeType curScopeType, const char * curScope, StatisticMeasure curMeasure, StatisticKind curKind, unsigned __int64 value) const
  3091. {
  3092. if ((curCreatorType != SCTall) && (creatorType != SCTall) && (creatorType != curCreatorType))
  3093. return false;
  3094. if ((curScopeType != SSTall) && (scopeType != SSTall) && (scopeType != curScopeType))
  3095. return false;
  3096. if ((curMeasure != SMeasureAll) && (measure != SMeasureAll) && (measure != curMeasure))
  3097. return false;
  3098. if ((curKind!= StKindAll) && (kind != StKindAll) && (kind != curKind))
  3099. return false;
  3100. if (!creatorFilter.match(curCreator))
  3101. return false;
  3102. if (!scopeFilter.match(curScope))
  3103. return false;
  3104. if (value != MaxStatisticValue)
  3105. {
  3106. if ((value < minValue) || (value > maxValue))
  3107. return false;
  3108. }
  3109. return true;
  3110. }
  3111. bool StatisticsFilter::recurseChildScopes(StatisticScopeType curScopeType, const char * curScope) const
  3112. {
  3113. switch (curScopeType)
  3114. {
  3115. case SSTgraph:
  3116. // A child of a graph will have depth 2 or more
  3117. if (!scopeFilter.matchDepth(2, (unsigned)-1))
  3118. return false;
  3119. break;
  3120. case SSTsubgraph:
  3121. // A child of a subgraph will have depth 3 or more
  3122. if (!scopeFilter.matchDepth(3, (unsigned)-1))
  3123. return false;
  3124. break;
  3125. }
  3126. if (!curScope)
  3127. return true;
  3128. return scopeFilter.recurseChildScopes(curScope);
  3129. }
  3130. void StatisticsFilter::set(const char * creatorTypeText, const char * scopeTypeText, const char * kindText)
  3131. {
  3132. StatisticCreatorType creatorType = queryCreatorType(creatorTypeText, SCTnone);
  3133. StatisticScopeType scopeType = queryScopeType(scopeTypeText, SSTnone);
  3134. if (creatorType != SCTnone)
  3135. setCreatorType(creatorType);
  3136. if (scopeType != SSTnone)
  3137. setScopeType(scopeType);
  3138. setKind(kindText);
  3139. }
  3140. void StatisticsFilter::set(const char * _creatorTypeText, const char * _creator, const char * _scopeTypeText, const char * _scope, const char * _measureText, const char * _kindText)
  3141. {
  3142. StatisticMeasure newMeasure = queryMeasure(_measureText, SMeasureNone);
  3143. if (newMeasure != SMeasureNone)
  3144. setMeasure(newMeasure);
  3145. set(_creatorTypeText, _scopeTypeText, _kindText);
  3146. setCreator(_creator);
  3147. setScope(_scope);
  3148. }
  3149. void StatisticsFilter::setCreatorDepth(unsigned _minCreatorDepth, unsigned _maxCreatorDepth)
  3150. {
  3151. creatorFilter.setDepth(_minCreatorDepth, _maxCreatorDepth);
  3152. }
  3153. void StatisticsFilter::setCreator(const char * _creator)
  3154. {
  3155. creatorFilter.set(_creator);
  3156. }
  3157. void StatisticsFilter::setCreatorType(StatisticCreatorType _creatorType)
  3158. {
  3159. creatorType = _creatorType;
  3160. }
  3161. void StatisticsFilter::addFilter(const char * filter)
  3162. {
  3163. //Match a filter of the form category[value] (use square brackets to avoid bash grief)
  3164. const char * openBra = strchr(filter, '[');
  3165. if (!openBra)
  3166. return;
  3167. const char * closeBra = strchr(openBra, ']');
  3168. if (!closeBra)
  3169. return;
  3170. const char * start = openBra + 1;
  3171. StringBuffer value(closeBra - start, start);
  3172. if (hasPrefix(filter, "creator[", false))
  3173. setCreator(value);
  3174. else if (hasPrefix(filter, "creatortype[", false))
  3175. setCreatorType(queryCreatorType(value, SCTall));
  3176. else if (hasPrefix(filter, "depth[", false))
  3177. {
  3178. const char * comma = strchr(value, ',');
  3179. if (comma)
  3180. setScopeDepth(atoi(value), atoi(comma+1));
  3181. else
  3182. setScopeDepth(atoi(value));
  3183. }
  3184. else if (hasPrefix(filter, "kind[", false))
  3185. setKind(value);
  3186. else if (hasPrefix(filter, "measure[", false))
  3187. setMeasure(queryMeasure(value, SMeasureAll));
  3188. else if (hasPrefix(filter, "scope[", false))
  3189. setScope(value);
  3190. else if (hasPrefix(filter, "scopetype[", false))
  3191. setScopeType(queryScopeType(value, SSTall));
  3192. else if (hasPrefix(filter, "value[", false))
  3193. {
  3194. //value[exact|low..high] where low and high are optional
  3195. unsigned __int64 lowValue = 0;
  3196. unsigned __int64 highValue = MaxStatisticValue;
  3197. if (isdigit(*value))
  3198. lowValue = (unsigned __int64)atoi64(value);
  3199. const char * dotdot = strstr(value, "..");
  3200. if (dotdot)
  3201. {
  3202. unsigned __int64 maxValue = (unsigned __int64)atoi64(dotdot + 2);
  3203. if (maxValue != 0)
  3204. highValue = maxValue;
  3205. }
  3206. else
  3207. highValue = lowValue;
  3208. setValueRange(lowValue, highValue);
  3209. }
  3210. else
  3211. throw MakeStringException(1, "Unknown stats filter '%s' - expected creator,creatortype,depth,kind,measure,scope,scopetype", filter);
  3212. }
  3213. void StatisticsFilter::setFilter(const char * filter)
  3214. {
  3215. if (isEmptyString(filter))
  3216. return;
  3217. for (;;)
  3218. {
  3219. const char * closeBra = strchr(filter, ']');
  3220. if (!closeBra)
  3221. throw MakeStringException(1, "Missing close bracket ']' in '%s' ", filter);
  3222. const char * comma = strchr(closeBra, ',');
  3223. if (comma)
  3224. {
  3225. //Take a copy - simplicity rather than efficiency
  3226. StringBuffer temp(comma - filter, filter);
  3227. addFilter(temp);
  3228. filter = comma + 1;
  3229. }
  3230. else
  3231. {
  3232. addFilter(filter);
  3233. return;
  3234. }
  3235. }
  3236. }
  3237. void StatisticsFilter::setScopeDepth(unsigned _scopeDepth)
  3238. {
  3239. scopeFilter.setDepth(_scopeDepth);
  3240. }
  3241. void StatisticsFilter::setScopeDepth(unsigned _minScopeDepth, unsigned _maxScopeDepth)
  3242. {
  3243. scopeFilter.setDepth(_minScopeDepth, _maxScopeDepth);
  3244. }
  3245. void StatisticsFilter::setScope(const char * _scope)
  3246. {
  3247. scopeFilter.set(_scope);
  3248. }
  3249. void StatisticsFilter::setScopeType(StatisticScopeType _scopeType)
  3250. {
  3251. scopeType = _scopeType;
  3252. switch (scopeType)
  3253. {
  3254. case SSTglobal:
  3255. case SSTgraph:
  3256. scopeFilter.setDepth(1);
  3257. break;
  3258. case SSTsubgraph:
  3259. scopeFilter.setDepth(2);
  3260. break;
  3261. case SSTactivity:
  3262. scopeFilter.setDepth(3);
  3263. break;
  3264. }
  3265. }
  3266. void StatisticsFilter::setMeasure(StatisticMeasure _measure)
  3267. {
  3268. measure = _measure;
  3269. }
  3270. void StatisticsFilter::setValueRange(unsigned __int64 _minValue, unsigned __int64 _maxValue)
  3271. {
  3272. minValue = _minValue;
  3273. maxValue = _maxValue;
  3274. }
  3275. void StatisticsFilter::setKind(StatisticKind _kind)
  3276. {
  3277. kind = _kind;
  3278. if (measure == SMeasureAll)
  3279. measure = queryMeasure(kind);
  3280. }
  3281. void StatisticsFilter::setKind(const char * _kind)
  3282. {
  3283. if (!_kind || !*_kind || streq(_kind, "*"))
  3284. {
  3285. if (measure == SMeasureNone)
  3286. measure = SMeasureAll;
  3287. kind = StKindAll;
  3288. return;
  3289. }
  3290. //Convert a kind wildcard to a measure
  3291. for (unsigned i1=SMeasureAll+1; i1 < SMeasureMax; i1++)
  3292. {
  3293. const char * prefix = queryMeasurePrefix((StatisticMeasure)i1);
  3294. size_t len = strlen(prefix);
  3295. if (len && strnicmp(_kind, prefix, len) == 0)
  3296. {
  3297. setMeasure((StatisticMeasure)i1);
  3298. //Treat When* and When as filters on times.
  3299. if (streq(_kind + len, "*") || !_kind[len])
  3300. return;
  3301. }
  3302. }
  3303. //Other wildcards not currently supported
  3304. kind = queryStatisticKind(_kind, StKindAll);
  3305. }
  3306. //---------------------------------------------------
  3307. class CStatsCategory : public CInterface
  3308. {
  3309. public:
  3310. StringAttr longName;
  3311. StringAttr shortName;
  3312. CStatsCategory(const char *_longName, const char *_shortName)
  3313. : longName(_longName), shortName(_shortName)
  3314. {
  3315. }
  3316. bool match(const char *_longName, const char *_shortName)
  3317. {
  3318. bool lm = stricmp(_longName, longName)==0;
  3319. bool sm = stricmp(_shortName, shortName)==0;
  3320. if (lm || sm)
  3321. {
  3322. if (lm && sm)
  3323. return true;
  3324. throw MakeStringException(0, "A stats category %s (%s) is already registered", shortName.get(), longName.get());
  3325. }
  3326. return false;
  3327. }
  3328. };
  3329. static CIArrayOf<CStatsCategory> statsCategories;
  3330. static CriticalSection statsCategoriesCrit;
  3331. extern int registerStatsCategory(const char *longName, const char *shortName)
  3332. {
  3333. CriticalBlock b(statsCategoriesCrit);
  3334. ForEachItemIn(idx, statsCategories)
  3335. {
  3336. if (statsCategories.item(idx).match(longName, shortName))
  3337. return idx;
  3338. }
  3339. statsCategories.append(*new CStatsCategory(longName, shortName));
  3340. return statsCategories.ordinality()-1;
  3341. }
  3342. static void checkKind(StatisticKind kind)
  3343. {
  3344. if (kind < StMax)
  3345. {
  3346. const StatisticMeta & meta = statsMetaData[kind];
  3347. if (meta.kind != kind)
  3348. throw makeStringExceptionV(0, "Statistic %u in the wrong order", kind);
  3349. }
  3350. StatisticKind serialKind = querySerializedKind(kind);
  3351. StatisticKind rawKind = queryRawKind(kind);
  3352. if (kind != serialKind)
  3353. assertex(queryRawKind(serialKind) == kind);
  3354. if (kind != rawKind)
  3355. assertex(querySerializedKind(rawKind) == kind);
  3356. StatisticMeasure measure = queryMeasure(kind);
  3357. const char * shortName = queryStatisticName(kind);
  3358. StringBuffer longName;
  3359. queryLongStatisticName(longName, kind);
  3360. const char * tagName __attribute__ ((unused)) = queryTreeTag(kind);
  3361. const char * prefix = queryMeasurePrefix(measure);
  3362. //Check short names are all correctly prefixed.
  3363. assertex(strncmp(shortName, prefix, strlen(prefix)) == 0);
  3364. }
  3365. static void checkDistributedKind(StatisticKind kind)
  3366. {
  3367. checkKind(kind);
  3368. checkKind((StatisticKind)(kind|StMinX));
  3369. checkKind((StatisticKind)(kind|StMaxX));
  3370. checkKind((StatisticKind)(kind|StAvgX));
  3371. checkKind((StatisticKind)(kind|StSkew));
  3372. checkKind((StatisticKind)(kind|StSkewMin));
  3373. checkKind((StatisticKind)(kind|StSkewMax));
  3374. checkKind((StatisticKind)(kind|StNodeMin));
  3375. checkKind((StatisticKind)(kind|StNodeMax));
  3376. checkKind((StatisticKind)(kind|StDeltaX));
  3377. }
  3378. void verifyStatisticFunctions()
  3379. {
  3380. static_assert(_elements_in(measureNames) == SMeasureMax+1 && !measureNames[SMeasureMax], "measureNames needs updating");
  3381. static_assert(_elements_in(creatorTypeNames) == SCTmax+1 && !creatorTypeNames[SCTmax], "creatorTypeNames needs updating");
  3382. static_assert(_elements_in(scopeTypeNames) == SSTmax+1 && !scopeTypeNames[SSTmax], "scopeTypeNames needs updating");
  3383. //Check the various functions return values for all possible values.
  3384. for (unsigned i1=SMeasureAll; i1 < SMeasureMax; i1++)
  3385. {
  3386. const char * prefix __attribute__((unused)) = queryMeasurePrefix((StatisticMeasure)i1);
  3387. const char * name = queryMeasureName((StatisticMeasure)i1);
  3388. assertex(queryMeasure(name, SMeasureMax) == i1);
  3389. }
  3390. for (StatisticScopeType sst = SSTnone; sst < SSTmax; sst = (StatisticScopeType)(sst+1))
  3391. {
  3392. const char * name = queryScopeTypeName(sst);
  3393. assertex(queryScopeType(name, SSTmax) == sst);
  3394. }
  3395. for (StatisticCreatorType sct = SCTnone; sct < SCTmax; sct = (StatisticCreatorType)(sct+1))
  3396. {
  3397. const char * name = queryCreatorTypeName(sct);
  3398. assertex(queryCreatorType(name, SCTmax) == sct);
  3399. }
  3400. for (unsigned i2=StKindAll+1; i2 < StMax; i2++)
  3401. {
  3402. checkDistributedKind((StatisticKind)i2);
  3403. }
  3404. }
  3405. #ifdef _DEBUG
  3406. MODULE_INIT(INIT_PRIORITY_STANDARD)
  3407. {
  3408. verifyStatisticFunctions();
  3409. return true;
  3410. }
  3411. #endif