jlibtests.cpp 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. /*
  14. * Jlib regression tests
  15. *
  16. */
  17. #ifdef _USE_CPPUNIT
  18. #include <memory>
  19. #include <chrono>
  20. #include "jsem.hpp"
  21. #include "jfile.hpp"
  22. #include "jdebug.hpp"
  23. #include "jset.hpp"
  24. #include "rmtfile.hpp"
  25. #include "jlzw.hpp"
  26. #include "jqueue.hpp"
  27. #include "jregexp.hpp"
  28. #include "jutil.hpp"
  29. #include "unittests.hpp"
  30. static const unsigned oneMinute = 60000; // msec
  31. class JlibSemTest : public CppUnit::TestFixture
  32. {
  33. public:
  34. CPPUNIT_TEST_SUITE(JlibSemTest);
  35. CPPUNIT_TEST(testSimple);
  36. CPPUNIT_TEST_SUITE_END();
  37. protected:
  38. void testTimedAvailable(Semaphore & sem)
  39. {
  40. //Shouldn't cause a reschedule, definitely shouldn't wait for 100s
  41. if(!sem.wait(100))
  42. ASSERT(false);
  43. }
  44. void testTimedElapsed(Semaphore & sem, unsigned time)
  45. {
  46. unsigned now = msTick();
  47. sem.wait(time);
  48. unsigned taken = msTick() - now;
  49. VStringBuffer errMsg("values: time: %u, taken: %u", time, taken);
  50. CPPUNIT_ASSERT_MESSAGE(errMsg.str(), taken >= time && taken < time + oneMinute);
  51. PROGLOG("%s", errMsg.str());
  52. }
  53. void testSimple()
  54. {
  55. //Some very basic semaphore tests.
  56. Semaphore sem;
  57. sem.signal();
  58. sem.wait();
  59. testTimedElapsed(sem, 100);
  60. sem.signal();
  61. testTimedAvailable(sem);
  62. sem.reinit(2);
  63. sem.wait();
  64. testTimedAvailable(sem);
  65. testTimedElapsed(sem, 5);
  66. }
  67. };
  68. CPPUNIT_TEST_SUITE_REGISTRATION( JlibSemTest );
  69. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibSemTest, "JlibSemTest" );
  70. class JlibSemTestStress : public CppUnit::TestFixture
  71. {
  72. public:
  73. CPPUNIT_TEST_SUITE(JlibSemTestStress);
  74. CPPUNIT_TEST(testSimple);
  75. CPPUNIT_TEST_SUITE_END();
  76. protected:
  77. void testTimedElapsed(Semaphore & sem, unsigned time, unsigned loopCount)
  78. {
  79. unsigned __int64 sumTaken = 0;
  80. unsigned maxTaken = 0;
  81. unsigned timeLimit = time + oneMinute;
  82. unsigned numberOfOut = 0;
  83. bool isSignaled = false;
  84. PROGLOG("Start loop");
  85. for (int i = 0 ; i <= loopCount; i++)
  86. {
  87. unsigned now = msTick();
  88. if (sem.wait(time))
  89. {
  90. isSignaled = true;
  91. break;
  92. }
  93. unsigned taken = msTick() - now;
  94. sumTaken += taken;
  95. maxTaken = (taken > maxTaken ? taken : maxTaken);
  96. numberOfOut += (taken > timeLimit ? 1 : 0);
  97. }
  98. VStringBuffer errMsg("values: time: %d, loop: %d, sum taken: %llu, average taken: %llu, max taken: %d, out of limit: %d times, signaled: %s",
  99. time, loopCount, sumTaken, sumTaken/loopCount, maxTaken, numberOfOut, (isSignaled ? "yes" : "no"));
  100. CPPUNIT_ASSERT_MESSAGE(errMsg.str(), 0 == numberOfOut && !isSignaled );
  101. PROGLOG("%s", errMsg.str());
  102. }
  103. void testSimple()
  104. {
  105. //Very basic semaphore stress tests.
  106. Semaphore sem;
  107. sem.signal();
  108. if (!sem.wait(1000))
  109. {
  110. VStringBuffer errMsg("Semaphore stalled (%s:%d)", sanitizeSourceFile(__FILE__), __LINE__);
  111. CPPUNIT_FAIL(errMsg.str());
  112. }
  113. testTimedElapsed(sem, 5, 1000);
  114. }
  115. };
  116. CPPUNIT_TEST_SUITE_REGISTRATION( JlibSemTestStress );
  117. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibSemTestStress, "JlibSemTestStress" );
  118. /* =========================================================== */
  119. class JlibSetTest : public CppUnit::TestFixture
  120. {
  121. protected:
  122. void testBitsetHelpers()
  123. {
  124. CPPUNIT_ASSERT_EQUAL(0U, countTrailingUnsetBits(1U));
  125. CPPUNIT_ASSERT_EQUAL(31U, countLeadingUnsetBits(1U));
  126. CPPUNIT_ASSERT_EQUAL(1U, getMostSignificantBit(1U));
  127. CPPUNIT_ASSERT_EQUAL(4U, countTrailingUnsetBits(0x110U));
  128. CPPUNIT_ASSERT_EQUAL(23U, countLeadingUnsetBits(0x110U));
  129. CPPUNIT_ASSERT_EQUAL(9U, getMostSignificantBit(0x110U));
  130. CPPUNIT_ASSERT_EQUAL(0U, countTrailingUnsetBits(0xFFFFFFFFU));
  131. CPPUNIT_ASSERT_EQUAL(0U, countLeadingUnsetBits(0xFFFFFFFFU));
  132. CPPUNIT_ASSERT_EQUAL(32U, getMostSignificantBit(0xFFFFFFFFU));
  133. CPPUNIT_ASSERT_EQUAL(52U, countTrailingUnsetBits(I64C(0x1010000000000000U)));
  134. }
  135. void testSet1(bool initial, IBitSet *bs, unsigned start, unsigned numBits, bool setValue, bool clearValue)
  136. {
  137. unsigned end = start+numBits;
  138. if (initial)
  139. bs->incl(start, end-1);
  140. for (unsigned i=start; i < end; i++)
  141. {
  142. ASSERT(bs->test(i) == clearValue);
  143. bs->set(i, setValue);
  144. ASSERT(bs->test(i) == setValue);
  145. bs->set(i+5, setValue);
  146. ASSERT(bs->scan(0, setValue) == i);
  147. ASSERT(bs->scan(i+1, setValue) == i+5);
  148. bs->set(i, clearValue);
  149. bs->set(i+5, clearValue);
  150. //Clearing i+5 above may extend the set - so need to calculate the end carefully
  151. unsigned last = i+5 < end ? end : i + 6;
  152. unsigned match1 = bs->scan(0, setValue);
  153. CPPUNIT_ASSERT_EQUAL((unsigned)(initial ? last : -1), match1);
  154. bs->invert(i);
  155. ASSERT(bs->test(i) == setValue);
  156. bs->invert(i);
  157. ASSERT(bs->test(i) == clearValue);
  158. bool wasSet = bs->testSet(i, setValue);
  159. ASSERT(wasSet == clearValue);
  160. bool wasSet2 = bs->testSet(i, clearValue);
  161. ASSERT(wasSet2 == setValue);
  162. ASSERT(bs->test(i) == clearValue);
  163. bs->set(i, setValue);
  164. unsigned match = bs->scanInvert(0, setValue);
  165. ASSERT(match == i);
  166. ASSERT(bs->test(i) == clearValue);
  167. }
  168. bs->reset();
  169. if (initial)
  170. {
  171. bs->incl(start, end);
  172. bs->excl(start+5, end-5);
  173. }
  174. else
  175. bs->incl(start+5, end-5);
  176. unsigned inclStart = bs->scan(start, setValue);
  177. ASSERT((start+5) == inclStart);
  178. unsigned inclEnd = bs->scan(start+5, clearValue);
  179. ASSERT((end-5) == (inclEnd-1));
  180. }
  181. void testSet(bool initial, unsigned passes, bool timed)
  182. {
  183. unsigned now = msTick();
  184. bool setValue = !initial;
  185. bool clearValue = initial;
  186. const unsigned numBits = 400;
  187. for (unsigned pass=0; pass < passes; pass++)
  188. {
  189. Owned<IBitSet> bs = createThreadSafeBitSet();
  190. testSet1(initial, bs, 0, numBits, setValue, clearValue);
  191. }
  192. if (timed)
  193. {
  194. unsigned elapsed = msTick()-now;
  195. DBGLOG("Bit test (%u) %d passes time taken = %dms", initial, passes, elapsed);
  196. }
  197. now = msTick();
  198. for (unsigned pass=0; pass < passes; pass++)
  199. {
  200. Owned<IBitSet> bs = createBitSet();
  201. testSet1(initial, bs, 0, numBits, setValue, clearValue);
  202. }
  203. if (timed)
  204. {
  205. unsigned elapsed = msTick()-now;
  206. DBGLOG("Bit test [thread-unsafe version] (%u) %d passes time taken = %dms", initial, passes, elapsed);
  207. }
  208. now = msTick();
  209. size32_t bitSetMemSz = getBitSetMemoryRequirement(numBits+5);
  210. MemoryBuffer mb;
  211. void *mem = mb.reserveTruncate(bitSetMemSz);
  212. for (unsigned pass=0; pass < passes; pass++)
  213. {
  214. Owned<IBitSet> bs = createBitSet(bitSetMemSz, mem);
  215. testSet1(initial, bs, 0, numBits, setValue, clearValue);
  216. }
  217. if (timed)
  218. {
  219. unsigned elapsed = msTick()-now;
  220. DBGLOG("Bit test [thread-unsafe version, fixed memory] (%u) %d passes time taken = %dms\n", initial, passes, elapsed);
  221. }
  222. }
  223. };
  224. class JlibSetTestQuick : public JlibSetTest
  225. {
  226. public:
  227. CPPUNIT_TEST_SUITE(JlibSetTestQuick);
  228. CPPUNIT_TEST(testBitsetHelpers);
  229. CPPUNIT_TEST(testSimple);
  230. CPPUNIT_TEST_SUITE_END();
  231. void testSimple()
  232. {
  233. testSet(false, 100, false);
  234. testSet(true, 100, false);
  235. }
  236. };
  237. class JlibSetTestStress : public JlibSetTest
  238. {
  239. public:
  240. CPPUNIT_TEST_SUITE(JlibSetTestStress);
  241. CPPUNIT_TEST(testParallel);
  242. CPPUNIT_TEST(testSimple);
  243. CPPUNIT_TEST_SUITE_END();
  244. void testSimple()
  245. {
  246. testSet(false, 10000, true);
  247. testSet(true, 10000, true);
  248. }
  249. protected:
  250. class CBitThread : public CSimpleInterfaceOf<IInterface>, implements IThreaded
  251. {
  252. IBitSet &bitSet;
  253. unsigned startBit, numBits;
  254. bool initial, setValue, clearValue;
  255. CThreaded threaded;
  256. Owned<IException> exception;
  257. CppUnit::Exception *cppunitException;
  258. public:
  259. CBitThread(IBitSet &_bitSet, unsigned _startBit, unsigned _numBits, bool _initial)
  260. : threaded("CBitThread", this), bitSet(_bitSet), startBit(_startBit), numBits(_numBits), initial(_initial)
  261. {
  262. cppunitException = NULL;
  263. setValue = !initial;
  264. clearValue = initial;
  265. }
  266. void start() { threaded.start(); }
  267. void join()
  268. {
  269. threaded.join();
  270. if (exception)
  271. throw exception.getClear();
  272. else if (cppunitException)
  273. throw cppunitException;
  274. }
  275. virtual void threadmain() override
  276. {
  277. try
  278. {
  279. unsigned endBit = startBit+numBits-1;
  280. if (initial)
  281. bitSet.incl(startBit, endBit);
  282. for (unsigned i=startBit; i < endBit; i++)
  283. {
  284. ASSERT(bitSet.test(i) == clearValue);
  285. bitSet.set(i, setValue);
  286. ASSERT(bitSet.test(i) == setValue);
  287. if (i < (endBit-1))
  288. ASSERT(bitSet.scan(i, clearValue) == i+1); // find next unset (should be i+1)
  289. bitSet.set(i, clearValue);
  290. bitSet.invert(i);
  291. ASSERT(bitSet.test(i) == setValue);
  292. bitSet.invert(i);
  293. ASSERT(bitSet.test(i) == clearValue);
  294. bool wasSet = bitSet.testSet(i, setValue);
  295. ASSERT(wasSet == clearValue);
  296. bool wasSet2 = bitSet.testSet(i, clearValue);
  297. ASSERT(wasSet2 == setValue);
  298. ASSERT(bitSet.test(i) == clearValue);
  299. bitSet.set(i, setValue);
  300. unsigned match = bitSet.scanInvert(startBit, setValue);
  301. ASSERT(match == i);
  302. ASSERT(bitSet.test(i) == clearValue);
  303. }
  304. }
  305. catch (IException *e)
  306. {
  307. exception.setown(e);
  308. }
  309. catch (CppUnit::Exception &e)
  310. {
  311. cppunitException = e.clone();
  312. }
  313. }
  314. };
  315. unsigned testParallelRun(IBitSet &bitSet, unsigned nThreads, unsigned bitsPerThread, bool initial)
  316. {
  317. IArrayOf<CBitThread> bitThreads;
  318. unsigned bitStart = 0;
  319. unsigned bitEnd = 0;
  320. for (unsigned t=0; t<nThreads; t++)
  321. {
  322. bitThreads.append(* new CBitThread(bitSet, bitStart, bitsPerThread, initial));
  323. bitStart += bitsPerThread;
  324. }
  325. unsigned now = msTick();
  326. for (unsigned t=0; t<nThreads; t++)
  327. bitThreads.item(t).start();
  328. Owned<IException> exception;
  329. CppUnit::Exception *cppunitException = NULL;
  330. for (unsigned t=0; t<nThreads; t++)
  331. {
  332. try
  333. {
  334. bitThreads.item(t).join();
  335. }
  336. catch (IException *e)
  337. {
  338. EXCLOG(e, NULL);
  339. if (!exception)
  340. exception.setown(e);
  341. else
  342. e->Release();
  343. }
  344. catch (CppUnit::Exception *e)
  345. {
  346. cppunitException = e;
  347. }
  348. }
  349. if (exception)
  350. throw exception.getClear();
  351. else if (cppunitException)
  352. throw *cppunitException;
  353. return msTick()-now;
  354. }
  355. void testSetParallel(bool initial)
  356. {
  357. unsigned numBits = 1000000; // 10M
  358. unsigned nThreads = getAffinityCpus();
  359. unsigned bitsPerThread = numBits/nThreads;
  360. bitsPerThread = ((bitsPerThread + (BitsPerItem-1)) / BitsPerItem) * BitsPerItem; // round up to multiple of BitsPerItem
  361. numBits = bitsPerThread*nThreads; // round
  362. fprintf(stdout, "testSetParallel, testing bit set of size : %d, nThreads=%d\n", numBits, nThreads);
  363. Owned<IBitSet> bitSet = createThreadSafeBitSet();
  364. unsigned took = testParallelRun(*bitSet, nThreads, bitsPerThread, initial);
  365. fprintf(stdout, "Thread safe parallel bit set test (%u) time taken = %dms\n", initial, took);
  366. size32_t bitSetMemSz = getBitSetMemoryRequirement(numBits);
  367. MemoryBuffer mb;
  368. void *mem = mb.reserveTruncate(bitSetMemSz);
  369. bitSet.setown(createBitSet(bitSetMemSz, mem));
  370. took = testParallelRun(*bitSet, nThreads, bitsPerThread, initial);
  371. fprintf(stdout, "Thread unsafe parallel bit set test (%u) time taken = %dms\n", initial, took);
  372. }
  373. void testParallel()
  374. {
  375. testSetParallel(false);
  376. testSetParallel(true);
  377. }
  378. };
  379. CPPUNIT_TEST_SUITE_REGISTRATION( JlibSetTestQuick );
  380. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibSetTestQuick, "JlibSetTestQuick" );
  381. CPPUNIT_TEST_SUITE_REGISTRATION( JlibSetTestStress );
  382. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibSetTestStress, "JlibSetTestStress" );
  383. /* =========================================================== */
  384. class JlibFileIOTestTiming : public CppUnit::TestFixture
  385. {
  386. protected:
  387. unsigned rs, nr10pct, nr150pct;
  388. char *record;
  389. StringBuffer tmpfile;
  390. CPPUNIT_TEST_SUITE( JlibFileIOTestTiming );
  391. CPPUNIT_TEST(testIOSmall);
  392. CPPUNIT_TEST(testIOLarge);
  393. CPPUNIT_TEST_SUITE_END();
  394. public:
  395. JlibFileIOTestTiming()
  396. {
  397. HardwareInfo hdwInfo;
  398. getHardwareInfo(hdwInfo);
  399. rs = 65536;
  400. unsigned nr = (unsigned)(1024.0 * (1024.0 * (double)hdwInfo.totalMemory / (double)rs));
  401. nr10pct = nr / 10;
  402. nr150pct = (unsigned)((double)nr * 1.5);
  403. record = (char *)malloc(rs);
  404. for (unsigned i=0;i<rs;i++)
  405. record[i] = 'a';
  406. record[rs-1] = '\n';
  407. tmpfile.set("JlibFileIOTest.txt");
  408. }
  409. ~JlibFileIOTestTiming()
  410. {
  411. free(record);
  412. }
  413. protected:
  414. void testIO(unsigned nr, const char *server)
  415. {
  416. IFile *ifile;
  417. IFileIO *ifileio;
  418. unsigned fsize = (unsigned)(((double)nr * (double)rs) / (1024.0 * 1024.0));
  419. fflush(NULL);
  420. fprintf(stdout,"\n");
  421. fflush(NULL);
  422. for(int j=0; j<2; j++)
  423. {
  424. if (j==0)
  425. fprintf(stdout, "File size: %d (MB) Cache, ", fsize);
  426. else
  427. fprintf(stdout, "\nFile size: %d (MB) Nocache, ", fsize);
  428. if (server != NULL)
  429. {
  430. SocketEndpoint ep;
  431. ep.set(server, 7100);
  432. ifile = createRemoteFile(ep, tmpfile);
  433. fprintf(stdout, "Remote: (%s)\n", server);
  434. }
  435. else
  436. {
  437. ifile = createIFile(tmpfile);
  438. fprintf(stdout, "Local:\n");
  439. }
  440. ifile->remove();
  441. unsigned st = msTick();
  442. IFEflags extraFlags = IFEcache;
  443. if (j==1)
  444. extraFlags = IFEnocache;
  445. ifileio = ifile->open(IFOcreate, extraFlags);
  446. #if 0 // for testing default and explicitly set share mode to Windows dafilesrv
  447. if (server != NULL)
  448. ifile->setShareMode((IFSHmode)IFSHread);
  449. #endif
  450. try
  451. {
  452. ifile->setFilePermissions(0666);
  453. }
  454. catch (...)
  455. {
  456. fprintf(stdout, "ifile->setFilePermissions() exception\n");
  457. }
  458. unsigned iter = nr / 40;
  459. if (iter < 1)
  460. iter = 1;
  461. __int64 pos = 0;
  462. for (unsigned i=0;i<nr;i++)
  463. {
  464. ifileio->write(pos, rs, record);
  465. pos += rs;
  466. if ((i % iter) == 0)
  467. {
  468. fprintf(stdout,".");
  469. fflush(NULL);
  470. }
  471. }
  472. ifileio->close();
  473. double rsec = (double)(msTick() - st)/1000.0;
  474. unsigned iorate = (unsigned)((double)fsize / rsec);
  475. fprintf(stdout, "\nwrite - elapsed time = %6.2f (s) iorate = %4d (MB/s)\n", rsec, iorate);
  476. st = msTick();
  477. extraFlags = IFEcache;
  478. if (j==1)
  479. extraFlags = IFEnocache;
  480. ifileio = ifile->open(IFOread, extraFlags);
  481. pos = 0;
  482. for (unsigned i=0;i<nr;i++)
  483. {
  484. ifileio->read(pos, rs, record);
  485. pos += rs;
  486. if ((i % iter) == 0)
  487. {
  488. fprintf(stdout,".");
  489. fflush(NULL);
  490. }
  491. }
  492. ifileio->close();
  493. rsec = (double)(msTick() - st)/1000.0;
  494. iorate = (unsigned)((double)fsize / rsec);
  495. fprintf(stdout, "\nread -- elapsed time = %6.2f (s) iorate = %4d (MB/s)\n", rsec, iorate);
  496. ifileio->Release();
  497. ifile->remove();
  498. ifile->Release();
  499. }
  500. }
  501. void testIOSmall()
  502. {
  503. testIO(nr10pct, NULL);
  504. }
  505. void testIOLarge()
  506. {
  507. testIO(nr150pct, NULL);
  508. }
  509. };
  510. class JlibFileIOTestStress : public JlibFileIOTestTiming
  511. {
  512. protected:
  513. CPPUNIT_TEST_SUITE( JlibFileIOTestStress );
  514. CPPUNIT_TEST(testIORemote);
  515. CPPUNIT_TEST_SUITE_END();
  516. void testIORemote()
  517. {
  518. const char * server = ".";
  519. testIO(nr10pct, server);
  520. }
  521. };
  522. CPPUNIT_TEST_SUITE_REGISTRATION( JlibFileIOTestTiming );
  523. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibFileIOTestTiming, "JlibFileIOTestTiming" );
  524. CPPUNIT_TEST_SUITE_REGISTRATION( JlibFileIOTestStress );
  525. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibFileIOTestTiming, "JlibFileIOTestStress" );
  526. /* =========================================================== */
  527. class JlibStringBufferTiming : public CppUnit::TestFixture
  528. {
  529. CPPUNIT_TEST_SUITE( JlibStringBufferTiming );
  530. CPPUNIT_TEST(testSwap);
  531. CPPUNIT_TEST_SUITE_END();
  532. public:
  533. void testSwap()
  534. {
  535. StringBuffer l;
  536. StringBuffer r;
  537. for (unsigned len=0; len<40; len++)
  538. {
  539. const unsigned numIter = 100000000;
  540. cycle_t start = get_cycles_now();
  541. for (unsigned pass=0; pass < numIter; pass++)
  542. {
  543. l.swapWith(r);
  544. }
  545. cycle_t elapsed = get_cycles_now() - start;
  546. DBGLOG("Each iteration of size %u took %.2f nanoseconds", len, (double)cycle_to_nanosec(elapsed) / numIter);
  547. l.append("a");
  548. r.append("b");
  549. }
  550. }
  551. };
  552. CPPUNIT_TEST_SUITE_REGISTRATION( JlibStringBufferTiming );
  553. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibStringBufferTiming, "JlibStringBufferTiming" );
  554. /* =========================================================== */
  555. static const unsigned split4_2[] = {0, 2, 4 };
  556. static const unsigned split100_2[] = {0, 50, 100 };
  557. static const unsigned split100_10[] = {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 };
  558. static const unsigned split7_10[] = {0,1,1,2,3,3,4,5,6,6,7 };
  559. static const unsigned split10_3[] = {0,3,7,10 };
  560. static const unsigned split58_10[] = {0,6,12,17,23,29,35,41,46,52,58 };
  561. static const unsigned split9_2T[] = { 0,5,9 };
  562. static const unsigned split9_2F[] = { 0,4,9 };
  563. static const unsigned split15_3[] = { 0,5,10,15 };
  564. class JlibQuantileTest : public CppUnit::TestFixture
  565. {
  566. CPPUNIT_TEST_SUITE( JlibQuantileTest );
  567. CPPUNIT_TEST(testQuantile);
  568. CPPUNIT_TEST(testRandom);
  569. CPPUNIT_TEST_SUITE_END();
  570. public:
  571. JlibQuantileTest()
  572. {
  573. }
  574. void testQuantilePos(unsigned numItems, unsigned numDivisions, bool roundUp, const unsigned * expected)
  575. {
  576. if (numDivisions == 0)
  577. return;
  578. QuantilePositionIterator iter(numItems, numDivisions, roundUp);
  579. QuantileFilterIterator filter(numItems, numDivisions, roundUp);
  580. unsigned prevPos = 0;
  581. iter.first();
  582. for (unsigned i=0; i <= numDivisions; i++)
  583. {
  584. //Check the values from the quantile iterator match those that are expected
  585. unsigned pos = (unsigned)iter.get();
  586. #if 0
  587. printf("(%d,%d) %d=%d\n", numItems, numDivisions, i, pos);
  588. #endif
  589. if (expected)
  590. CPPUNIT_ASSERT_EQUAL(expected[i], pos);
  591. //Check that the quantile filter correctly returns true and false for subsequent calls.
  592. while (prevPos < pos)
  593. {
  594. CPPUNIT_ASSERT(!filter.get());
  595. filter.next();
  596. prevPos++;
  597. }
  598. if (prevPos == pos)
  599. {
  600. CPPUNIT_ASSERT(filter.get());
  601. filter.next();
  602. prevPos++;
  603. }
  604. iter.next();
  605. }
  606. }
  607. void testQuantile()
  608. {
  609. testQuantilePos(4, 2, false, split4_2);
  610. testQuantilePos(100, 2, false, split100_2);
  611. testQuantilePos(100, 10, false, split100_10);
  612. testQuantilePos(7, 10, false, split7_10);
  613. testQuantilePos(10, 3, false, split10_3);
  614. testQuantilePos(10, 3, true, split10_3);
  615. testQuantilePos(58, 10, false, split58_10);
  616. //testQuantilePos(9, 2, true, split9_2T);
  617. testQuantilePos(9, 2, false, split9_2F);
  618. testQuantilePos(15, 3, false, split15_3);
  619. testQuantilePos(1231, 57, false, NULL);
  620. testQuantilePos(1, 63, false, NULL);
  621. testQuantilePos(10001, 17, false, NULL);
  622. }
  623. void testRandom()
  624. {
  625. //test various random combinations to ensure the results are consistent.
  626. for (unsigned i=0; i < 10; i++)
  627. testQuantilePos(random() % 1000000, random() % 10000, true, NULL);
  628. }
  629. };
  630. CPPUNIT_TEST_SUITE_REGISTRATION( JlibQuantileTest );
  631. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibQuantileTest, "JlibQuantileTest" );
  632. /* =========================================================== */
  633. class JlibTimingTest : public CppUnit::TestFixture
  634. {
  635. CPPUNIT_TEST_SUITE( JlibTimingTest );
  636. CPPUNIT_TEST(testMsTick);
  637. CPPUNIT_TEST(testGetCyclesNow);
  638. CPPUNIT_TEST(testStdChrono);
  639. CPPUNIT_TEST(testGetTimeOfDay);
  640. CPPUNIT_TEST(testClockGetTimeReal);
  641. CPPUNIT_TEST(testClockGetTimeMono);
  642. CPPUNIT_TEST(testTimestampNow);
  643. CPPUNIT_TEST_SUITE_END();
  644. public:
  645. static constexpr unsigned scale = 10;
  646. static constexpr unsigned iters = 1000000 * scale;
  647. JlibTimingTest()
  648. {
  649. }
  650. void testMsTick()
  651. {
  652. unsigned startTime = msTick();
  653. unsigned value = 0;
  654. for (unsigned i=0; i < iters; i++)
  655. value += msTick();
  656. printf("msTick() %uns = %u\n", (msTick()-startTime)/scale, value);
  657. }
  658. void testGetCyclesNow()
  659. {
  660. unsigned startTime = msTick();
  661. unsigned value = 0;
  662. for (unsigned i=0; i < iters; i++)
  663. value += get_cycles_now();
  664. printf("get_cycles_now() %uns = %u\n", (msTick()-startTime)/scale, value);
  665. }
  666. void testStdChrono()
  667. {
  668. unsigned startTime = msTick();
  669. unsigned value = 0;
  670. for (unsigned i=0; i < iters; i++)
  671. value += std::chrono::high_resolution_clock::now().time_since_epoch().count();
  672. printf("std::chrono::high_resolution_clock::now() %uns = %u\n", (msTick()-startTime)/scale, value);
  673. }
  674. void testGetTimeOfDay()
  675. {
  676. unsigned startTime = msTick();
  677. struct timeval tv;
  678. unsigned value = 0;
  679. for (unsigned i=0; i < iters; i++)
  680. {
  681. gettimeofday(&tv, NULL);
  682. value += tv.tv_sec;
  683. }
  684. printf("gettimeofday() %uns = %u\n", (msTick()-startTime)/scale, value);
  685. }
  686. void testClockGetTimeReal()
  687. {
  688. unsigned startTime = msTick();
  689. struct timespec ts;
  690. unsigned value = 0;
  691. for (unsigned i=0; i < iters; i++)
  692. {
  693. clock_gettime(CLOCK_REALTIME, &ts);
  694. value += ts.tv_sec;
  695. }
  696. printf("clock_gettime(REALTIME) %uns = %u\n", (msTick()-startTime)/scale, value);
  697. }
  698. void testClockGetTimeMono()
  699. {
  700. unsigned startTime = msTick();
  701. struct timespec ts;
  702. unsigned value = 0;
  703. for (unsigned i=0; i < iters; i++)
  704. {
  705. clock_gettime(CLOCK_MONOTONIC, &ts);
  706. value += ts.tv_sec;
  707. }
  708. printf("clock_gettime(MONOTONIC) %uns = %u\n", (msTick()-startTime)/scale, value);
  709. }
  710. void testTimestampNow()
  711. {
  712. unsigned startTime = msTick();
  713. struct timespec ts;
  714. unsigned value = 0;
  715. for (unsigned i=0; i < iters; i++)
  716. {
  717. value += getTimeStampNowValue();
  718. }
  719. printf("getTimeStampNowValue() %uns = %u\n", (msTick()-startTime)/scale, value);
  720. }
  721. };
  722. CPPUNIT_TEST_SUITE_REGISTRATION( JlibTimingTest );
  723. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibTimingTest, "JlibTimingTest" );
  724. /* =========================================================== */
  725. class JlibReaderWriterTestTiming : public CppUnit::TestFixture
  726. {
  727. CPPUNIT_TEST_SUITE(JlibReaderWriterTestTiming);
  728. CPPUNIT_TEST(testCombinations);
  729. CPPUNIT_TEST_SUITE_END();
  730. const static unsigned spinScaling = 1000;
  731. static unsigned spinCalculation(unsigned prev, unsigned scale)
  732. {
  733. unsigned value = prev;
  734. for (unsigned i = 0; i < scale*spinScaling; i++)
  735. {
  736. value = (value * 0x1234FEDB + 0x87654321);
  737. }
  738. return value;
  739. }
  740. class Reader : public Thread
  741. {
  742. public:
  743. Reader(IRowQueue & _source, Semaphore & _doneSem, unsigned _workScale)
  744. : Thread("Reader"), source(_source), doneSem(_doneSem), workScale(_workScale), work(0)
  745. {
  746. }
  747. virtual int run()
  748. {
  749. for (;;)
  750. {
  751. const void * next;
  752. if (!source.dequeue(next))
  753. break;
  754. if (!next)
  755. break;
  756. std::atomic<byte> * value = (std::atomic<byte> *)next;
  757. (*value)++;
  758. if (workScale)
  759. work = spinCalculation(work, workScale);
  760. }
  761. doneSem.signal();
  762. return 0;
  763. }
  764. private:
  765. IRowQueue & source;
  766. Semaphore & doneSem;
  767. volatile unsigned work;
  768. unsigned workScale;
  769. };
  770. class WriterBase : public Thread
  771. {
  772. public:
  773. WriterBase(IRowQueue & _target, size_t _len, byte * _buffer, Semaphore & _startSem, Semaphore & _doneSem, unsigned _workScale)
  774. : Thread("Writer"), target(_target), len(_len), buffer(_buffer), startSem(_startSem), doneSem(_doneSem), workScale(_workScale), work(0)
  775. {
  776. }
  777. protected:
  778. size_t len;
  779. byte * buffer;
  780. IRowQueue & target;
  781. Semaphore & startSem;
  782. Semaphore & doneSem;
  783. volatile unsigned work;
  784. unsigned workScale;
  785. };
  786. class Writer : public WriterBase
  787. {
  788. public:
  789. Writer(IRowQueue & _target, size_t _len, byte * _buffer, Semaphore & _startSem, Semaphore & _doneSem, unsigned _workScale)
  790. : WriterBase(_target, _len, _buffer, _startSem, _doneSem, _workScale)
  791. {
  792. }
  793. virtual int run()
  794. {
  795. startSem.wait();
  796. for (size_t i = 0; i < len; i++)
  797. {
  798. if (workScale)
  799. work = spinCalculation(work, workScale);
  800. target.enqueue(buffer + i);
  801. }
  802. target.noteWriterStopped();
  803. doneSem.signal();
  804. return 0;
  805. }
  806. };
  807. public:
  808. const static size_t bufferSize = 0x100000;//0x100000*64;
  809. void testQueue(IRowQueue & queue, unsigned numProducers, unsigned numConsumers, unsigned queueElements, unsigned readerWork, unsigned writerWork)
  810. {
  811. const size_t sizePerProducer = bufferSize / numProducers;
  812. const size_t testSize = sizePerProducer * numProducers;
  813. OwnedMalloc<byte> buffer(bufferSize, true);
  814. Semaphore startSem;
  815. Semaphore writerDoneSem;
  816. Semaphore stopSem;
  817. Reader * * consumers = new Reader *[numConsumers];
  818. for (unsigned i2 = 0; i2 < numConsumers; i2++)
  819. {
  820. consumers[i2] = new Reader(queue, stopSem, readerWork);
  821. consumers[i2]->start();
  822. }
  823. WriterBase * * producers = new WriterBase *[numProducers];
  824. for (unsigned i1 = 0; i1 < numProducers; i1++)
  825. {
  826. producers[i1] = new Writer(queue, sizePerProducer, buffer + i1 * sizePerProducer, startSem, writerDoneSem, writerWork);
  827. producers[i1]->start();
  828. }
  829. cycle_t startTime = get_cycles_now();
  830. //Start the writers
  831. startSem.signal(numProducers);
  832. //Wait for the writers to complete
  833. for (unsigned i7 = 0; i7 < numProducers; i7++)
  834. writerDoneSem.wait();
  835. //Wait for the readers to complete
  836. for (unsigned i3 = 0; i3 < numConsumers; i3++)
  837. stopSem.wait();
  838. cycle_t stopTime = get_cycles_now();
  839. //All bytes should have been changed to 1, if not a queue item got lost.
  840. unsigned failures = 0;
  841. unsigned numClear = 0;
  842. size_t failPos = ~(size_t)0;
  843. byte failValue = 0;
  844. for (size_t pos = 0; pos < testSize; pos++)
  845. {
  846. if (buffer[pos] != 1)
  847. {
  848. failures++;
  849. if (failPos == ~(size_t)0)
  850. {
  851. failPos = pos;
  852. failValue = buffer[pos];
  853. }
  854. }
  855. if (buffer[pos] == 0)
  856. numClear++;
  857. }
  858. unsigned timeMs = cycle_to_nanosec(stopTime - startTime) / 1000000;
  859. unsigned expectedReadWorkTime = (unsigned)(((double)unitWorkTimeMs * readerWork) / numConsumers);
  860. unsigned expectedWriteWorkTime = (unsigned)(((double)unitWorkTimeMs * writerWork) / numProducers);
  861. unsigned expectedWorkTime = std::max(expectedReadWorkTime, expectedWriteWorkTime);
  862. if (failures)
  863. {
  864. printf("Fail: Test %u producers %u consumers %u queueItems %u(%u) mismatches fail(@%u=%u)\n", numProducers, numConsumers, queueElements, failures, numClear, (unsigned)failPos, failValue);
  865. ASSERT(failures == 0);
  866. }
  867. else
  868. printf("Pass: Test %u(@%u) producers %u(@%u) consumers %u queueItems in %ums [%dms]\n", numProducers, writerWork, numConsumers, readerWork, queueElements, timeMs, timeMs-expectedWorkTime);
  869. for (unsigned i4 = 0; i4 < numConsumers; i4++)
  870. {
  871. consumers[i4]->join();
  872. consumers[i4]->Release();
  873. }
  874. delete[] consumers;
  875. for (unsigned i5 = 0; i5 < numProducers; i5++)
  876. {
  877. producers[i5]->join();
  878. producers[i5]->Release();
  879. }
  880. delete[] producers;
  881. }
  882. void testQueue(unsigned numProducers, unsigned numConsumers, unsigned numElements = 0, unsigned readWork = 0, unsigned writeWork = 0)
  883. {
  884. unsigned queueElements = (numElements != 0) ? numElements : (numProducers + numConsumers) * 2;
  885. Owned<IRowQueue> queue = createRowQueue(numConsumers, numProducers, queueElements, 0);
  886. testQueue(*queue, numProducers, numConsumers, queueElements, readWork, writeWork);
  887. }
  888. void testWorkQueue(unsigned numProducers, unsigned numConsumers, unsigned numElements)
  889. {
  890. for (unsigned readWork = 1; readWork <= 8; readWork = readWork * 2)
  891. {
  892. for (unsigned writeWork = 1; writeWork <= 8; writeWork = writeWork * 2)
  893. {
  894. testQueue(numProducers, numConsumers, numElements, readWork, writeWork);
  895. }
  896. }
  897. }
  898. void testCombinations()
  899. {
  900. // 1:1
  901. for (unsigned i=0; i < 10; i++)
  902. testQueue(1, 1, 10);
  903. //One to Many
  904. testQueue(1, 10, 5);
  905. testQueue(1, 5, 5);
  906. testQueue(1, 5, 10);
  907. testQueue(1, 127, 10);
  908. testQueue(1, 127, 127);
  909. //Many to One
  910. testQueue(10, 1, 5);
  911. testQueue(5, 1, 5);
  912. testQueue(5, 1, 10);
  913. testQueue(127, 1, 127);
  914. cycle_t startTime = get_cycles_now();
  915. volatile unsigned value = 0;
  916. for (unsigned pass = 0; pass < 10; pass++)
  917. {
  918. for (unsigned i2 = 0; i2 < bufferSize; i2++)
  919. value = spinCalculation(value, 1);
  920. }
  921. cycle_t stopTime = get_cycles_now();
  922. unitWorkTimeMs = cycle_to_nanosec(stopTime - startTime) / (1000000 * 10);
  923. printf("Work(1) takes %ums\n", unitWorkTimeMs);
  924. //How does it scale with number of queue elements?
  925. for (unsigned elem = 16; elem < 256; elem *= 2)
  926. {
  927. testQueue(16, 1, elem, 1, 1);
  928. }
  929. #if 1
  930. //Many to Many
  931. for (unsigned readWork = 1; readWork <= 8; readWork = readWork * 2)
  932. {
  933. for (unsigned writeWork = 1; writeWork <= 8; writeWork = writeWork * 2)
  934. {
  935. testQueue(1, 1, 63, readWork, writeWork);
  936. testQueue(1, 2, 63, readWork, writeWork);
  937. testQueue(1, 4, 63, readWork, writeWork);
  938. testQueue(1, 8, 63, readWork, writeWork);
  939. testQueue(1, 16, 63, readWork, writeWork);
  940. testQueue(2, 1, 63, readWork, writeWork);
  941. testQueue(4, 1, 63, readWork, writeWork);
  942. testQueue(8, 1, 63, readWork, writeWork);
  943. testQueue(16, 1, 63, readWork, writeWork);
  944. testQueue(2, 2, 63, readWork, writeWork);
  945. testQueue(4, 4, 63, readWork, writeWork);
  946. testQueue(8, 8, 63, readWork, writeWork);
  947. testQueue(16, 8, 63, readWork, writeWork);
  948. testQueue(16, 16, 63, readWork, writeWork);
  949. testQueue(32, 1, 63, readWork, writeWork);
  950. testQueue(64, 1, 63, readWork, writeWork);
  951. testQueue(1, 32, 63, readWork, writeWork);
  952. testQueue(1, 64, 63, readWork, writeWork);
  953. }
  954. }
  955. #else
  956. //Many to Many
  957. testWorkQueue(1, 1, 63);
  958. testWorkQueue(1, 2, 63);
  959. testWorkQueue(1, 4, 63);
  960. testWorkQueue(1, 8, 63);
  961. testWorkQueue(1, 16, 63);
  962. testWorkQueue(2, 1, 63);
  963. testWorkQueue(4, 1, 63);
  964. testWorkQueue(8, 1, 63);
  965. testWorkQueue(16, 1, 63);
  966. testWorkQueue(2, 2, 63);
  967. testWorkQueue(4, 4, 63);
  968. testWorkQueue(8, 8, 63);
  969. #endif
  970. testQueue(2, 2, 4);
  971. testQueue(2, 2, 8);
  972. testQueue(2, 2, 16);
  973. testQueue(2, 2, 32);
  974. testQueue(2, 2, 100);
  975. }
  976. protected:
  977. unsigned unitWorkTimeMs;
  978. };
  979. CPPUNIT_TEST_SUITE_REGISTRATION(JlibReaderWriterTestTiming);
  980. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibReaderWriterTestTiming, "JlibReaderWriterTestTiming");
  981. /* =========================================================== */
  982. class JlibWildMatchBase : public CppUnit::TestFixture
  983. {
  984. protected:
  985. void testSet(unsigned length, const char * const * patterns, bool reportTiming)
  986. {
  987. std::unique_ptr<char[]> search(generateSearchString(length));
  988. CCycleTimer timer;
  989. testPatterns(search.get(), patterns);
  990. if (reportTiming)
  991. printf("%u: %u ms\n", length, timer.elapsedMs());
  992. }
  993. char * generateSearchString(size_t len)
  994. {
  995. char * target = new char[len+1];
  996. fillSearchString(target, len);
  997. target[len] = 0;
  998. return target;
  999. }
  1000. void fillSearchString(char * target, size_t len)
  1001. {
  1002. for (unsigned repeat=0; ; repeat++)
  1003. {
  1004. for (unsigned char fill = 'a'; fill <= 'z'; fill++)
  1005. {
  1006. for (unsigned i=0; i < repeat; i++)
  1007. {
  1008. *target++ = fill;
  1009. if (--len == 0)
  1010. return;
  1011. }
  1012. }
  1013. }
  1014. }
  1015. void testPatterns(const char * search, const char * const * patterns)
  1016. {
  1017. for (const char * const * cur = patterns; *cur; cur++)
  1018. {
  1019. const char * pattern = *cur;
  1020. bool expected = true;
  1021. bool nocase = false;
  1022. if (*pattern == '!')
  1023. {
  1024. expected = false;
  1025. pattern++;
  1026. }
  1027. if (*pattern == '~')
  1028. {
  1029. nocase = true;
  1030. pattern++;
  1031. }
  1032. bool evaluated = WildMatch(search, pattern, nocase);
  1033. CPPUNIT_ASSERT_EQUAL_MESSAGE(pattern, expected, evaluated);
  1034. }
  1035. }
  1036. };
  1037. const char * const patterns10 [] = {
  1038. "!a",
  1039. "abcdefghij",
  1040. "??????????",
  1041. "?*c?*e*",
  1042. "!??*b?*h*",
  1043. "a*",
  1044. "*j",
  1045. "a*j",
  1046. "a**j",
  1047. "a***************j",
  1048. "abcde*fghij",
  1049. "!abcde*?*fghij",
  1050. "*a*j*",
  1051. "*a*c*e*g*j*",
  1052. "a?c?e?g??j",
  1053. "a?c?e?g?*?j",
  1054. "!~A",
  1055. "!A*",
  1056. "~A*",
  1057. "~*J",
  1058. "~A*J",
  1059. "~A**J",
  1060. "~A***************J",
  1061. "~*A*J*",
  1062. "~*A*C*E*G*J*",
  1063. "~*A*B*C*D*E*F*G*H*I*J*",
  1064. "~*A*?*?*?*J*",
  1065. "~*A*?C*?E*?*J*",
  1066. "~*A*C?*E?*?*J*",
  1067. "!~*A*.B*C*D*E*F*G*H*I*J*",
  1068. nullptr
  1069. };
  1070. const char * const patterns100 [] = {
  1071. "a*",
  1072. "*h",
  1073. "a*h",
  1074. "a**h",
  1075. "a***************h",
  1076. "*a*j*",
  1077. "*a*c*e*g*j*",
  1078. "!a*jj*fff",
  1079. "!a*jj*zzz",
  1080. "a*jj*fff*",
  1081. "*aa*jj*fff*",
  1082. "!a*jj*zy*",
  1083. nullptr
  1084. };
  1085. const char * const patternsLarge [] = {
  1086. "!*a*zy*",
  1087. "a*",
  1088. "a*h*",
  1089. "!a*jj*ab",
  1090. "!a*jj*zy",
  1091. "a*jj*fff*",
  1092. "!a*jj*zy*",
  1093. /* "!a*c*e*g*i*k*zy*", will completely destroy the performance*/
  1094. nullptr
  1095. };
  1096. class JlibWildMatchCore : public JlibWildMatchBase
  1097. {
  1098. CPPUNIT_TEST_SUITE(JlibWildMatchCore);
  1099. CPPUNIT_TEST(testWildMatch);
  1100. CPPUNIT_TEST_SUITE_END();
  1101. public:
  1102. void testWildMatch()
  1103. {
  1104. testSet(10, patterns10, false);
  1105. testSet(100, patterns100, false);
  1106. testSet(1000, patternsLarge, false);
  1107. }
  1108. };
  1109. CPPUNIT_TEST_SUITE_REGISTRATION(JlibWildMatchCore);
  1110. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibWildMatchCore, "JlibWildMatchCore");
  1111. class JlibWildMatchTiming : public JlibWildMatchBase
  1112. {
  1113. CPPUNIT_TEST_SUITE(JlibWildMatchTiming);
  1114. CPPUNIT_TEST(testWildMatch);
  1115. CPPUNIT_TEST_SUITE_END();
  1116. public:
  1117. void testWildMatch()
  1118. {
  1119. testSet(10000, patternsLarge, true);
  1120. testSet(100000, patternsLarge, true);
  1121. testSet(1000000, patternsLarge, true);
  1122. }
  1123. };
  1124. CPPUNIT_TEST_SUITE_REGISTRATION(JlibWildMatchTiming);
  1125. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibWildMatchTiming, "JlibWildMatchTiming");
  1126. const EnumMapping mapping[] = {
  1127. { 1, "one" },
  1128. { 3, "three" },
  1129. { 5, "five" },
  1130. {0, nullptr }
  1131. };
  1132. const char * strings[] = { "zero", "one", "two", "three", "four", nullptr };
  1133. class JlibMapping : public CppUnit::TestFixture
  1134. {
  1135. CPPUNIT_TEST_SUITE(JlibMapping);
  1136. CPPUNIT_TEST(testEnum);
  1137. CPPUNIT_TEST(testMatch);
  1138. CPPUNIT_TEST_SUITE_END();
  1139. public:
  1140. void testEnum()
  1141. {
  1142. CPPUNIT_ASSERT(streq("one", getEnumText(1, mapping)));
  1143. CPPUNIT_ASSERT(streq("three", getEnumText(3, mapping)));
  1144. CPPUNIT_ASSERT(streq("five", getEnumText(5, mapping)));
  1145. CPPUNIT_ASSERT(streq("two", getEnumText(2, mapping, "two")));
  1146. CPPUNIT_ASSERT(!getEnumText(2, mapping, nullptr));
  1147. CPPUNIT_ASSERT_EQUAL(1, getEnum("one", mapping));
  1148. CPPUNIT_ASSERT_EQUAL(3, getEnum("three", mapping));
  1149. CPPUNIT_ASSERT_EQUAL(5, getEnum("five", mapping));
  1150. CPPUNIT_ASSERT_EQUAL(99, getEnum("seven", mapping, 99));
  1151. }
  1152. void testMatch()
  1153. {
  1154. CPPUNIT_ASSERT_EQUAL(0U, matchString("zero", strings));
  1155. CPPUNIT_ASSERT_EQUAL(1U, matchString("one", strings));
  1156. CPPUNIT_ASSERT_EQUAL(4U, matchString("four", strings));
  1157. CPPUNIT_ASSERT_EQUAL(UINT_MAX, matchString("ten", strings));
  1158. }
  1159. };
  1160. CPPUNIT_TEST_SUITE_REGISTRATION(JlibMapping);
  1161. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibMapping, "JlibMapping");
  1162. class JlibIPTTest : public CppUnit::TestFixture
  1163. {
  1164. CPPUNIT_TEST_SUITE(JlibIPTTest);
  1165. CPPUNIT_TEST(test);
  1166. CPPUNIT_TEST(testMarkup);
  1167. CPPUNIT_TEST(testRootArrayMarkup);
  1168. CPPUNIT_TEST(testArrayMarkup);
  1169. CPPUNIT_TEST(testMergeConfig);
  1170. CPPUNIT_TEST(testRemoveReuse);
  1171. CPPUNIT_TEST_SUITE_END();
  1172. public:
  1173. void testArrayMarkup()
  1174. {
  1175. static constexpr const char * yamlFlowMarkup = R"!!({a: {
  1176. b: valb,
  1177. c: [valc],
  1178. d: [vald1,vald2],
  1179. e: [{x: valex1, y: valey1}],
  1180. f: {x: valfx1, y: valfy1},
  1181. g: [{x: valgx1, y: valgy1},{x: valgx2, y: valgy2}],
  1182. h: !el valh,
  1183. i: {
  1184. j: {
  1185. b: valb,
  1186. c: [valc],
  1187. d: [vald1,vald2],
  1188. e: [{x: valex1, y: valey1}],
  1189. f: {x: valfx1, y: valfy1},
  1190. g: [{x: valgx1, y: valgy1},{x: valgx2, y: valgy2}],
  1191. h: !el valh,
  1192. },
  1193. k: [{
  1194. b: valb,
  1195. c: [valc],
  1196. d: [vald1,vald2],
  1197. e: [{x: valex1, y: valey1}],
  1198. f: {x: valfx1, y: valfy1},
  1199. g: [{x: valgx1, y: valgy1},{x: valgx2, y: valgy2}],
  1200. h: !el valh,
  1201. }],
  1202. l: [{
  1203. b: valb,
  1204. c: [valc],
  1205. d: [vald1,vald2],
  1206. e: [{x: valex1, y: valey1}],
  1207. f: {x: valfx1, y: valfy1},
  1208. g: [{x: valgx1, y: valgy1},{x: valgx2, y: valgy2}],
  1209. h: !el valh,
  1210. },
  1211. {
  1212. b: valb,
  1213. c: [valc],
  1214. d: [vald1,vald2],
  1215. e: [{x: valex1, y: valey1}],
  1216. f: {x: valfx1, y: valfy1},
  1217. g: [{x: valgx1, y: valgy1},{x: valgx2, y: valgy2}],
  1218. h: !el valh,
  1219. }],
  1220. }
  1221. }
  1222. }
  1223. )!!";
  1224. static constexpr const char * yamlBlockMarkup = R"!!(a:
  1225. b: valb
  1226. c:
  1227. - valc
  1228. d:
  1229. - vald1
  1230. - vald2
  1231. e:
  1232. - x: valex1
  1233. y: valey1
  1234. f:
  1235. x: valfx1
  1236. y: valfy1
  1237. g:
  1238. - x: valgx1
  1239. y: valgy1
  1240. - x: valgx2
  1241. y: valgy2
  1242. h: !el valh
  1243. i:
  1244. j:
  1245. b: valb
  1246. c:
  1247. - valc
  1248. d:
  1249. - vald1
  1250. - vald2
  1251. e:
  1252. - x: valex1
  1253. y: valey1
  1254. f:
  1255. x: valfx1
  1256. y: valfy1
  1257. g:
  1258. - x: valgx1
  1259. y: valgy1
  1260. - x: valgx2
  1261. y: valgy2
  1262. h: !el valh
  1263. k:
  1264. - b: valb
  1265. c:
  1266. - valc
  1267. d:
  1268. - vald1
  1269. - vald2
  1270. e:
  1271. - x: valex1
  1272. y: valey1
  1273. f:
  1274. x: valfx1
  1275. y: valfy1
  1276. g:
  1277. - x: valgx1
  1278. y: valgy1
  1279. - x: valgx2
  1280. y: valgy2
  1281. h: !el valh
  1282. l:
  1283. - b: valb
  1284. c:
  1285. - valc
  1286. d:
  1287. - vald1
  1288. - vald2
  1289. e:
  1290. - x: valex1
  1291. y: valey1
  1292. f:
  1293. x: valfx1
  1294. y: valfy1
  1295. g:
  1296. - x: valgx1
  1297. y: valgy1
  1298. - x: valgx2
  1299. y: valgy2
  1300. h: !el valh
  1301. - b: valb
  1302. c:
  1303. - valc
  1304. d:
  1305. - vald1
  1306. - vald2
  1307. e:
  1308. - x: valex1
  1309. y: valey1
  1310. f:
  1311. x: valfx1
  1312. y: valfy1
  1313. g:
  1314. - x: valgx1
  1315. y: valgy1
  1316. - x: valgx2
  1317. y: valgy2
  1318. h: !el valh
  1319. )!!";
  1320. StringBuffer ml;
  1321. Owned<IPropertyTree> yamlFlow = createPTreeFromYAMLString(yamlFlowMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1322. toYAML(yamlFlow, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
  1323. CPPUNIT_ASSERT(streq(ml, yamlBlockMarkup));
  1324. Owned<IPropertyTree> yamlBlock = createPTreeFromYAMLString(yamlBlockMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1325. toYAML(yamlBlock, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
  1326. CPPUNIT_ASSERT(streq(ml, yamlBlockMarkup));
  1327. }
  1328. void testMergeConfig()
  1329. {
  1330. static constexpr const char * yamlLeft = R"!!({a: {
  1331. b: gone,
  1332. bk: kept,
  1333. c: [gone],
  1334. ck: [kept],
  1335. d: [gone1,gone2],
  1336. dk: [kept1,kept2],
  1337. e: [{name: merged, x: gone, z: kept},{altname: merged, x: gone, z: kept},{name: kept, x: kept, y: kept}, {unnamed: kept, x: kept, y: kept}],
  1338. ek: [{name: kept, x: kept, y: kept}, {unnamed: kept, x: kept, y: kept}],
  1339. f: [{unnamed: gone, x: gone, y: gone}, {unnamed: gone2, x: gone2, y: gone2}],
  1340. kept: {x: kept, y: kept},
  1341. merged: {x: gone, z: kept}
  1342. }
  1343. }
  1344. )!!";
  1345. static constexpr const char * yamlBlockLeft = R"!!(a:
  1346. b: gone
  1347. bk: kept
  1348. c:
  1349. - gone
  1350. ck:
  1351. - kept
  1352. d:
  1353. - gone1
  1354. - gone2
  1355. dk:
  1356. - kept1
  1357. - kept2
  1358. e:
  1359. - name: merged
  1360. x: gone
  1361. z: kept
  1362. - altname: merged
  1363. x: gone
  1364. z: kept
  1365. - name: kept
  1366. x: kept
  1367. y: kept
  1368. - unnamed: kept
  1369. x: kept
  1370. y: kept
  1371. ek:
  1372. - name: kept
  1373. x: kept
  1374. y: kept
  1375. - unnamed: kept
  1376. x: kept
  1377. y: kept
  1378. f:
  1379. - unnamed: gone
  1380. x: gone
  1381. y: gone
  1382. - unnamed: gone2
  1383. x: gone2
  1384. y: gone2
  1385. kept:
  1386. x: kept
  1387. y: kept
  1388. merged:
  1389. x: gone
  1390. z: kept
  1391. )!!";
  1392. static constexpr const char * yamlRight = R"!!({a: {
  1393. b: updated,
  1394. c: [added],
  1395. d: [added1,added2],
  1396. e: [{name: merged, x: updated, y: added},{altname: merged, x: updated, y: added},{name: added, x: added, y: added}, {unnamed: added, x: added, y: added}],
  1397. f: [{unnamed: kept, x: kept, y: kept}, {unnamed: kept2, x: kept2, y: kept2}],
  1398. added: {x: added, y: added},
  1399. merged: {x: updated, y: added}
  1400. }
  1401. }
  1402. )!!";
  1403. static constexpr const char * yamlBlockRight = R"!!(a:
  1404. b: updated
  1405. c:
  1406. - added
  1407. d:
  1408. - added1
  1409. - added2
  1410. e:
  1411. - name: merged
  1412. x: updated
  1413. y: added
  1414. - altname: merged
  1415. x: updated
  1416. y: added
  1417. - name: added
  1418. x: added
  1419. y: added
  1420. - unnamed: added
  1421. x: added
  1422. y: added
  1423. f:
  1424. - unnamed: kept
  1425. x: kept
  1426. y: kept
  1427. - unnamed: kept2
  1428. x: kept2
  1429. y: kept2
  1430. added:
  1431. x: added
  1432. y: added
  1433. merged:
  1434. x: updated
  1435. y: added
  1436. )!!";
  1437. static constexpr const char * yamlMerged = R"!!(a:
  1438. b: updated
  1439. bk: kept
  1440. added:
  1441. x: added
  1442. y: added
  1443. c:
  1444. - added
  1445. ck:
  1446. - kept
  1447. d:
  1448. - added1
  1449. - added2
  1450. dk:
  1451. - kept1
  1452. - kept2
  1453. e:
  1454. - name: merged
  1455. x: updated
  1456. y: added
  1457. z: kept
  1458. - altname: merged
  1459. x: updated
  1460. y: added
  1461. z: kept
  1462. - name: kept
  1463. x: kept
  1464. y: kept
  1465. - unnamed: kept
  1466. x: kept
  1467. y: kept
  1468. - name: added
  1469. x: added
  1470. y: added
  1471. - unnamed: added
  1472. x: added
  1473. y: added
  1474. ek:
  1475. - name: kept
  1476. x: kept
  1477. y: kept
  1478. - unnamed: kept
  1479. x: kept
  1480. y: kept
  1481. f:
  1482. - unnamed: kept
  1483. x: kept
  1484. y: kept
  1485. - unnamed: kept2
  1486. x: kept2
  1487. y: kept2
  1488. kept:
  1489. x: kept
  1490. y: kept
  1491. merged:
  1492. x: updated
  1493. y: added
  1494. z: kept
  1495. )!!";
  1496. StringBuffer ml;
  1497. Owned<IPropertyTree> treeLeft = createPTreeFromYAMLString(yamlLeft, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1498. Owned<IPropertyTree> treeRight = createPTreeFromYAMLString(yamlRight, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1499. mergeConfiguration(*treeLeft, *treeRight, "@altname");
  1500. toYAML(treeLeft, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
  1501. CPPUNIT_ASSERT(streq(ml, yamlMerged));
  1502. Owned<IPropertyTree> treeBlockLeft = createPTreeFromYAMLString(yamlBlockLeft, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1503. Owned<IPropertyTree> treeBlockRight = createPTreeFromYAMLString(yamlBlockRight, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1504. mergeConfiguration(*treeBlockLeft, *treeBlockRight, "@altname");
  1505. toYAML(treeBlockLeft, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
  1506. CPPUNIT_ASSERT(streq(ml, yamlMerged));
  1507. }
  1508. void testRootArrayMarkup()
  1509. {
  1510. static constexpr const char * xmlMarkup = R"!!(<__array__>
  1511. <__item__ a="val1a" b="val2a"/>
  1512. <__item__ a="val1b" b="val2b"/>
  1513. <__item__ a="val1c" b="val2c"/>
  1514. </__array__>
  1515. )!!";
  1516. static constexpr const char * jsonMarkup = R"!!([
  1517. {
  1518. "@a": "val1a",
  1519. "@b": "val2a"
  1520. },
  1521. {
  1522. "@a": "val1b",
  1523. "@b": "val2b"
  1524. },
  1525. {
  1526. "@a": "val1c",
  1527. "@b": "val2c"
  1528. }
  1529. ])!!";
  1530. static constexpr const char * yamlMarkup = R"!!(- a: val1a
  1531. b: val2a
  1532. - a: val1b
  1533. b: val2b
  1534. - a: val1c
  1535. b: val2c
  1536. )!!";
  1537. Owned<IPropertyTree> xml = createPTreeFromXMLString(xmlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1538. Owned<IPropertyTree> yaml = createPTreeFromYAMLString(yamlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1539. Owned<IPropertyTree> json = createPTreeFromJSONString(jsonMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1540. CPPUNIT_ASSERT(areMatchingPTrees(xml, json));
  1541. CPPUNIT_ASSERT(areMatchingPTrees(xml, yaml));
  1542. StringBuffer ml;
  1543. toXML(xml, ml, 0, XML_Format|XML_SortTags);
  1544. CPPUNIT_ASSERT(streq(ml, xmlMarkup));
  1545. toYAML(xml, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
  1546. CPPUNIT_ASSERT(streq(ml, yamlMarkup));
  1547. toJSON(xml, ml.clear(), 0, JSON_Format|JSON_SortTags|JSON_HideRootArrayObject);
  1548. CPPUNIT_ASSERT(streq(ml, jsonMarkup));
  1549. }
  1550. void testMarkup()
  1551. {
  1552. static constexpr const char * xmlMarkup = R"!!( <__object__ attr1="attrval1" attr2="attrval2">
  1553. <binmixed bin="1" xsi:type="SOAP-ENC:base64">
  1554. CwAAAA== </binmixed>
  1555. <binsimple xsi:type="SOAP-ENC:base64">
  1556. CwAAAA== </binsimple>
  1557. <element1>scalarvalue</element1>
  1558. <item a="1"
  1559. b="2"
  1560. c="3"
  1561. d="4"/>
  1562. <item a="2"/>
  1563. <item a="3"/>
  1564. <scalars>
  1565. <valX>x</valX>
  1566. <valX>x</valX>
  1567. <valY>y</valY>
  1568. <valY>y</valY>
  1569. <valZ>z</valZ>
  1570. </scalars>
  1571. <sub1 subattr1="sav1">
  1572. sub1val
  1573. </sub1>
  1574. <sub2 subattr2="sav2">
  1575. sub2val
  1576. </sub2>
  1577. <subX subattr3="sav3">
  1578. subXval
  1579. </subX>
  1580. cpptestval
  1581. </__object__>
  1582. )!!";
  1583. static constexpr const char * yamlMarkup = R"!!(attr1: attrval1
  1584. attr2: attrval2
  1585. binmixed:
  1586. bin: 1
  1587. ^: !binary |-
  1588. CwAAAA==
  1589. binsimple: !binary |-
  1590. CwAAAA==
  1591. element1: !el scalarvalue
  1592. item:
  1593. - a: 1
  1594. b: 2
  1595. c: 3
  1596. d: 4
  1597. - a: 2
  1598. - a: 3
  1599. scalars:
  1600. valX:
  1601. - x
  1602. - x
  1603. valY:
  1604. - y
  1605. - y
  1606. valZ: !el z
  1607. sub1:
  1608. subattr1: sav1
  1609. ^: !el sub1val
  1610. sub2:
  1611. subattr2: sav2
  1612. ^: !el sub2val
  1613. subX:
  1614. subattr3: sav3
  1615. ^: !el subXval
  1616. ^: !el cpptestval
  1617. )!!";
  1618. static constexpr const char * jsonMarkup = R"!!({
  1619. "@attr1": "attrval1",
  1620. "@attr2": "attrval2",
  1621. "binmixed": {
  1622. "@bin": "1",
  1623. "#valuebin": "CwAAAA=="
  1624. },
  1625. "binsimple": {
  1626. "#valuebin": "CwAAAA=="
  1627. },
  1628. "element1": "scalarvalue",
  1629. "item": [
  1630. {
  1631. "@a": "1",
  1632. "@b": "2",
  1633. "@c": "3",
  1634. "@d": "4"
  1635. },
  1636. {
  1637. "@a": "2"
  1638. },
  1639. {
  1640. "@a": "3"
  1641. }
  1642. ],
  1643. "scalars": {
  1644. "valX": [
  1645. "x",
  1646. "x"
  1647. ],
  1648. "valY": [
  1649. "y",
  1650. "y"
  1651. ],
  1652. "valZ": "z"
  1653. },
  1654. "sub1": {
  1655. "@subattr1": "sav1",
  1656. "#value": "sub1val"
  1657. },
  1658. "sub2": {
  1659. "@subattr2": "sav2",
  1660. "#value": "sub2val"
  1661. },
  1662. "subX": {
  1663. "@subattr3": "sav3",
  1664. "#value": "subXval"
  1665. },
  1666. "#value": "cpptestval"
  1667. })!!";
  1668. Owned<IPropertyTree> xml = createPTreeFromXMLString(xmlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1669. Owned<IPropertyTree> yaml = createPTreeFromYAMLString(yamlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1670. Owned<IPropertyTree> json = createPTreeFromJSONString(jsonMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
  1671. CPPUNIT_ASSERT(areMatchingPTrees(xml, yaml));
  1672. CPPUNIT_ASSERT(areMatchingPTrees(xml, json));
  1673. //if we want the final compares to be less fragile (test will have to be updated if formatting changes) we could reparse and compare trees again
  1674. StringBuffer ml;
  1675. toXML(xml, ml, 2, XML_Format|XML_SortTags);
  1676. CPPUNIT_ASSERT(streq(ml, xmlMarkup));
  1677. toYAML(yaml, ml.clear(), 2, YAML_SortTags|YAML_HideRootArrayObject);
  1678. CPPUNIT_ASSERT(streq(ml, yamlMarkup));
  1679. toJSON(json, ml.clear(), 2, JSON_Format|JSON_SortTags|JSON_HideRootArrayObject);
  1680. CPPUNIT_ASSERT(streq(ml, jsonMarkup));
  1681. }
  1682. void test()
  1683. {
  1684. Owned<IPropertyTree> testTree = createPTreeFromXMLString(
  1685. "<cpptest attr1='attrval1' attr2='attrval2'>"
  1686. " <sub1 subattr1='sav1'>sub1val</sub1>"
  1687. " <sub2 subattr2='sav2'>sub2val</sub2>"
  1688. " <subX subattr3='sav3'>subXval</subX>"
  1689. " <item a='1' b='2' c='3' d='4'/>"
  1690. " <item a='2'/>"
  1691. " <item a='3'/>"
  1692. " <array>"
  1693. " <valX>x</valX>"
  1694. " <valX>x</valX>"
  1695. " <valY>y</valY>"
  1696. " <valY>y</valY>"
  1697. " <valZ>z</valZ>"
  1698. " </array>"
  1699. " <binprop bin='1' xsi:type='SOAP-ENC:base64'>CwAAAA==</binprop>"
  1700. "cpptestval"
  1701. "</cpptest>");
  1702. MemoryBuffer mb;
  1703. mb.reserveTruncate(4*1024+1); // Must be > PTREE_COMPRESS_THRESHOLD (see top of jptree.cpp)
  1704. testTree->addProp("binprop/subbinprop", "nonbinval1");
  1705. testTree->addPropBin("binprop/subbinprop", mb.length(), mb.toByteArray());
  1706. testTree->addProp("binprop/subbinprop", "nonbinval2");
  1707. testTree->addPropBin("binprop/subbinprop", mb.length(), mb.toByteArray());
  1708. // test some sets in prep. for 'get' tests
  1709. CPPUNIT_ASSERT(testTree->renameProp("subX", "subY"));
  1710. IPropertyTree *subY = testTree->queryPropTree("subY");
  1711. CPPUNIT_ASSERT(testTree->renameTree(subY, "sub3"));
  1712. IPropertyTree *subtest = testTree->setPropTree("subtest");
  1713. subtest = testTree->addPropTree("subtest", createPTree());
  1714. CPPUNIT_ASSERT(subtest != nullptr);
  1715. subtest = testTree->queryPropTree("subtest[2]");
  1716. CPPUNIT_ASSERT(subtest != nullptr);
  1717. subtest->setProp("str", "str1");
  1718. subtest->addProp("str", "str2");
  1719. subtest->appendProp("str[2]", "-more");
  1720. subtest->setPropBool("bool", true);
  1721. subtest->addPropBool("bool", false);
  1722. subtest->setPropInt("int", 1);
  1723. subtest->addPropInt("int", 2);
  1724. subtest->setPropInt64("int64", 1);
  1725. subtest->addPropInt64("int64", 2);
  1726. mb.clear().append("binstr1");
  1727. subtest->setPropBin("bin", mb.length(), mb.toByteArray());
  1728. mb.clear().append("binstr2");
  1729. subtest->addPropBin("bin", mb.length(), mb.toByteArray());
  1730. mb.clear().append("-more");
  1731. subtest->appendPropBin("bin[1]", mb.length(), mb.toByteArray());
  1732. // position insertion.
  1733. testTree->addProp("newprop", "v1");
  1734. testTree->addProp("newprop", "v2");
  1735. testTree->addProp("newprop[2]", "v3");
  1736. CPPUNIT_ASSERT(streq("v3", testTree->queryProp("newprop[2]")));
  1737. CPPUNIT_ASSERT(testTree->hasProp("sub1"));
  1738. CPPUNIT_ASSERT(testTree->hasProp("sub1/@subattr1"));
  1739. CPPUNIT_ASSERT(testTree->hasProp("sub2/@subattr2"));
  1740. CPPUNIT_ASSERT(testTree->hasProp("@attr1"));
  1741. CPPUNIT_ASSERT(!testTree->isBinary("@attr1"));
  1742. CPPUNIT_ASSERT(!testTree->isBinary("sub1"));
  1743. CPPUNIT_ASSERT(testTree->isBinary("binprop"));
  1744. CPPUNIT_ASSERT(!testTree->isCompressed("binprop"));
  1745. CPPUNIT_ASSERT(!testTree->isBinary("binprop/subbinprop[1]"));
  1746. CPPUNIT_ASSERT(testTree->isBinary("binprop/subbinprop[2]"));
  1747. CPPUNIT_ASSERT(!testTree->isCompressed("binprop/subbinprop[3]"));
  1748. CPPUNIT_ASSERT(testTree->isCompressed("binprop/subbinprop[4]"));
  1749. // testing if subX was renamed correctly
  1750. CPPUNIT_ASSERT(!testTree->hasProp("subX"));
  1751. CPPUNIT_ASSERT(!testTree->hasProp("subY"));
  1752. CPPUNIT_ASSERT(testTree->hasProp("sub3"));
  1753. StringBuffer astr;
  1754. CPPUNIT_ASSERT(testTree->getProp("sub1", astr));
  1755. CPPUNIT_ASSERT(streq("sub1val", astr.str()));
  1756. CPPUNIT_ASSERT(streq("sub2val", testTree->queryProp("sub2")));
  1757. subtest = testTree->queryPropTree("subtest[2]");
  1758. CPPUNIT_ASSERT(subtest != nullptr);
  1759. CPPUNIT_ASSERT(subtest->getPropBool("bool[1]"));
  1760. CPPUNIT_ASSERT(!subtest->getPropBool("bool[2]"));
  1761. CPPUNIT_ASSERT(1 == subtest->getPropInt("int[1]"));
  1762. CPPUNIT_ASSERT(2 == subtest->getPropInt("int[2]"));
  1763. CPPUNIT_ASSERT(1 == subtest->getPropInt64("int64[1]"));
  1764. CPPUNIT_ASSERT(2 == subtest->getPropInt64("int64[2]"));
  1765. subtest->getPropBin("bin[1]", mb.clear());
  1766. const char *ptr = (const char *)mb.toByteArray();
  1767. CPPUNIT_ASSERT(streq("binstr1", ptr)); // NB: null terminator was added at set time.
  1768. CPPUNIT_ASSERT(streq("-more", ptr+strlen("binstr1")+1)); // NB: null terminator was added at set time.
  1769. subtest->getPropBin("bin[2]", mb.clear());
  1770. CPPUNIT_ASSERT(streq("binstr2", mb.toByteArray())); // NB: null terminator was added at set time.
  1771. CPPUNIT_ASSERT(testTree->hasProp("subtest/bin[2]"));
  1772. CPPUNIT_ASSERT(testTree->removeProp("subtest/bin[2]"));
  1773. CPPUNIT_ASSERT(!testTree->hasProp("subtest/bin[2]"));
  1774. CPPUNIT_ASSERT(testTree->hasProp("subtest"));
  1775. CPPUNIT_ASSERT(testTree->removeTree(subtest)); // this is subtest[2]
  1776. subtest = testTree->queryPropTree("subtest"); // now just 1
  1777. CPPUNIT_ASSERT(testTree->removeTree(subtest));
  1778. CPPUNIT_ASSERT(!testTree->hasProp("subtest"));
  1779. IPropertyTree *item3 = testTree->queryPropTree("item[@a='3']");
  1780. CPPUNIT_ASSERT(nullptr != item3);
  1781. CPPUNIT_ASSERT(2 == testTree->queryChildIndex(item3));
  1782. CPPUNIT_ASSERT(streq("item", item3->queryName()));
  1783. Owned<IPropertyTreeIterator> iter = testTree->getElements("item");
  1784. unsigned a=1;
  1785. ForEach(*iter)
  1786. {
  1787. CPPUNIT_ASSERT(a == iter->query().getPropInt("@a"));
  1788. ++a;
  1789. }
  1790. Owned<IAttributeIterator> attrIter = testTree->queryPropTree("item[1]")->getAttributes();
  1791. CPPUNIT_ASSERT(4 == attrIter->count());
  1792. unsigned i = 0;
  1793. ForEach(*attrIter)
  1794. {
  1795. const char *name = attrIter->queryName();
  1796. const char *val = attrIter->queryValue();
  1797. CPPUNIT_ASSERT('a'+i == *(name+1));
  1798. CPPUNIT_ASSERT('1'+i == *val);
  1799. ++i;
  1800. }
  1801. IPropertyTree *array = testTree->queryPropTree("array");
  1802. CPPUNIT_ASSERT(array != nullptr);
  1803. CPPUNIT_ASSERT(array->hasChildren());
  1804. CPPUNIT_ASSERT(3 == array->numUniq());
  1805. CPPUNIT_ASSERT(5 == array->numChildren());
  1806. CPPUNIT_ASSERT(!testTree->isCaseInsensitive());
  1807. CPPUNIT_ASSERT(3 == testTree->getCount("sub*"));
  1808. testTree->addPropInt("newitem", 1);
  1809. testTree->addPropInt("newitem", 2);
  1810. testTree->addPropInt("./newitem", 3);
  1811. testTree->addPropInt("././newitem", 4);
  1812. Owned<IPropertyTreeIterator> xIter = testTree->getElements("./newitem");
  1813. unsigned match=1;
  1814. ForEach(*xIter)
  1815. {
  1816. CPPUNIT_ASSERT(match == xIter->query().getPropInt(nullptr));
  1817. ++match;
  1818. }
  1819. }
  1820. void testRemoveReuse()
  1821. {
  1822. Owned<IPropertyTree> t = createPTree();
  1823. t->addPropInt("a", 1);
  1824. t->addPropInt("a", 2);
  1825. Owned<IPropertyTree> a1 = t->getPropTree("a[1]");
  1826. Owned<IPropertyTree> a2 = t->getPropTree("a[2]");
  1827. CPPUNIT_ASSERT(t->removeProp("a[2]"));
  1828. CPPUNIT_ASSERT(t->removeProp("a"));
  1829. CPPUNIT_ASSERT(!a1->isArray(nullptr));
  1830. CPPUNIT_ASSERT(!a2->isArray(nullptr));
  1831. IPropertyTree *na2 = t->addPropTree("na", a2.getClear());
  1832. CPPUNIT_ASSERT(!na2->isArray(nullptr));
  1833. IPropertyTree *na1 = t->addPropTree("na", a1.getClear());
  1834. CPPUNIT_ASSERT(na1->isArray(nullptr));
  1835. CPPUNIT_ASSERT(na2->isArray(nullptr));
  1836. }
  1837. };
  1838. CPPUNIT_TEST_SUITE_REGISTRATION(JlibIPTTest);
  1839. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibIPTTest, "JlibIPTTest");
  1840. #include "jdebug.hpp"
  1841. #include "jmutex.hpp"
  1842. class AtomicTimingTest : public CppUnit::TestFixture
  1843. {
  1844. CPPUNIT_TEST_SUITE(AtomicTimingTest);
  1845. CPPUNIT_TEST(runAllTests);
  1846. CPPUNIT_TEST_SUITE_END();
  1847. public:
  1848. class CasCounter
  1849. {
  1850. public:
  1851. CasCounter() = default;
  1852. CasCounter(unsigned __int64 _value) : value{_value} {}
  1853. operator unsigned __int64() { return value; }
  1854. unsigned __int64 operator = (unsigned __int64 _value)
  1855. {
  1856. value = _value;
  1857. return _value;
  1858. }
  1859. unsigned __int64 operator ++(int)
  1860. {
  1861. unsigned __int64 expected = value.load();
  1862. while (!value.compare_exchange_weak(expected, expected + 1))
  1863. {
  1864. }
  1865. return expected+1;
  1866. }
  1867. std::atomic<unsigned __int64> value = { 0 };
  1868. };
  1869. template <typename LOCK, typename BLOCK, typename COUNTER, unsigned NUMVALUES, unsigned NUMLOCKS>
  1870. class LockTester
  1871. {
  1872. public:
  1873. LockTester()
  1874. {
  1875. value1 = 0;
  1876. }
  1877. class LockTestThread : public Thread
  1878. {
  1879. public:
  1880. LockTestThread(Semaphore & _startSem, Semaphore & _endSem, LOCK & _lock1, COUNTER & _value1, LOCK & _lock2, COUNTER * _extraValues, unsigned _numIterations)
  1881. : startSem(_startSem), endSem(_endSem),
  1882. lock1(_lock1), value1(_value1),
  1883. lock2(_lock2), extraValues(_extraValues),
  1884. numIterations(_numIterations)
  1885. {
  1886. }
  1887. virtual void execute()
  1888. {
  1889. {
  1890. BLOCK block(lock1);
  1891. value1++;
  1892. if (NUMVALUES >= 2)
  1893. extraValues[1]++;
  1894. if (NUMVALUES >= 3)
  1895. extraValues[2]++;
  1896. if (NUMVALUES >= 4)
  1897. extraValues[3]++;
  1898. if (NUMVALUES >= 5)
  1899. extraValues[4]++;
  1900. }
  1901. if (NUMLOCKS == 2)
  1902. {
  1903. BLOCK block(lock2);
  1904. extraValues[1]++;
  1905. }
  1906. }
  1907. virtual int run()
  1908. {
  1909. startSem.wait();
  1910. for (unsigned i = 0; i < numIterations; i++)
  1911. execute();
  1912. endSem.signal();
  1913. return 0;
  1914. }
  1915. protected:
  1916. Semaphore & startSem;
  1917. Semaphore & endSem;
  1918. LOCK & lock1;
  1919. LOCK & lock2;
  1920. COUNTER & value1;
  1921. COUNTER * extraValues;
  1922. const unsigned numIterations;
  1923. };
  1924. unsigned __int64 run(const char * title, unsigned numThreads, unsigned numIterations)
  1925. {
  1926. value1 = 0;
  1927. for (unsigned ix = 1; ix < NUMVALUES; ix++)
  1928. extraValues[ix] = 0;
  1929. for (unsigned i = 0; i < numThreads; i++)
  1930. {
  1931. LockTestThread * next = new LockTestThread(startSem, endSem, lock, value1, lock, extraValues, numIterations);
  1932. threads.append(*next);
  1933. next->start();
  1934. }
  1935. cycle_t startCycles = get_cycles_now();
  1936. startSem.signal(numThreads);
  1937. for (unsigned i2 = 0; i2 < numThreads; i2++)
  1938. endSem.wait();
  1939. cycle_t endCycles = get_cycles_now();
  1940. unsigned __int64 expected = (unsigned __int64)numIterations * numThreads;
  1941. unsigned __int64 averageTime = cycle_to_nanosec(endCycles - startCycles) / (numIterations * numThreads);
  1942. printf("%s@%u/%u threads(%u) %" I64F "uns/iteration lost(%" I64F "d)\n", title, NUMVALUES, NUMLOCKS, numThreads, averageTime, expected - value1);
  1943. for (unsigned i3 = 0; i3 < numThreads; i3++)
  1944. threads.item(i3).join();
  1945. return averageTime;
  1946. }
  1947. protected:
  1948. CIArrayOf<LockTestThread> threads;
  1949. Semaphore startSem;
  1950. Semaphore endSem;
  1951. LOCK lock;
  1952. COUNTER value1;
  1953. COUNTER extraValues[NUMVALUES];
  1954. };
  1955. #define DO_TEST(LOCK, CLOCK, COUNTER, NUMVALUES, NUMLOCKS) \
  1956. { \
  1957. const char * title = #LOCK "," #COUNTER;\
  1958. LockTester<LOCK, CLOCK, COUNTER, NUMVALUES, NUMLOCKS> tester;\
  1959. uncontendedTimes.append(tester.run(title, 1, numIterations));\
  1960. minorTimes.append(tester.run(title, 2, numIterations));\
  1961. typicalTimes.append(tester.run(title, numCores / 2, numIterations));\
  1962. tester.run(title, numCores, numIterations);\
  1963. tester.run(title, numCores + 1, numIterations);\
  1964. contendedTimes.append(tester.run(title, numCores * 2, numIterations));\
  1965. }
  1966. //Use to common out a test
  1967. #define XDO_TEST(LOCK, CLOCK, COUNTER, NUMVALUES, NUMLOCKS) \
  1968. { \
  1969. uncontendedTimes.append(0);\
  1970. minorTimes.append(0);\
  1971. typicalTimes.append(0);\
  1972. contendedTimes.append(0);\
  1973. }
  1974. class Null
  1975. {};
  1976. const unsigned numIterations = 1000000;
  1977. const unsigned numCores = getAffinityCpus();
  1978. void runAllTests()
  1979. {
  1980. DO_TEST(CriticalSection, CriticalBlock, unsigned __int64, 1, 1);
  1981. DO_TEST(CriticalSection, CriticalBlock, unsigned __int64, 2, 1);
  1982. DO_TEST(CriticalSection, CriticalBlock, unsigned __int64, 5, 1);
  1983. DO_TEST(CriticalSection, CriticalBlock, unsigned __int64, 1, 2);
  1984. DO_TEST(SpinLock, SpinBlock, unsigned __int64, 1, 1);
  1985. DO_TEST(SpinLock, SpinBlock, unsigned __int64, 2, 1);
  1986. DO_TEST(SpinLock, SpinBlock, unsigned __int64, 5, 1);
  1987. DO_TEST(SpinLock, SpinBlock, unsigned __int64, 1, 2);
  1988. DO_TEST(Null, Null, std::atomic<unsigned __int64>, 1, 1);
  1989. DO_TEST(Null, Null, std::atomic<unsigned __int64>, 2, 1);
  1990. DO_TEST(Null, Null, std::atomic<unsigned __int64>, 5, 1);
  1991. DO_TEST(Null, Null, std::atomic<unsigned __int64>, 1, 2);
  1992. DO_TEST(Null, Null, RelaxedAtomic<unsigned __int64>, 1, 1);
  1993. DO_TEST(Null, Null, RelaxedAtomic<unsigned __int64>, 5, 1);
  1994. DO_TEST(Null, Null, CasCounter, 1, 1);
  1995. DO_TEST(Null, Null, CasCounter, 5, 1);
  1996. DO_TEST(Null, Null, unsigned __int64, 1, 1);
  1997. DO_TEST(Null, Null, unsigned __int64, 2, 1);
  1998. DO_TEST(Null, Null, unsigned __int64, 5, 1);
  1999. //Read locks will fail to prevent values being lost, but the timings are useful in comparison with CriticalSection
  2000. DO_TEST(ReadWriteLock, ReadLockBlock, unsigned __int64, 1, 1);
  2001. DO_TEST(ReadWriteLock, ReadLockBlock, unsigned __int64, 2, 1);
  2002. DO_TEST(ReadWriteLock, ReadLockBlock, unsigned __int64, 5, 1);
  2003. DO_TEST(ReadWriteLock, ReadLockBlock, unsigned __int64, 1, 2);
  2004. DO_TEST(ReadWriteLock, WriteLockBlock, unsigned __int64, 1, 1);
  2005. DO_TEST(ReadWriteLock, WriteLockBlock, unsigned __int64, 2, 1);
  2006. DO_TEST(ReadWriteLock, WriteLockBlock, unsigned __int64, 5, 1);
  2007. DO_TEST(ReadWriteLock, WriteLockBlock, unsigned __int64, 1, 2);
  2008. printf("Summary\n");
  2009. summariseTimings("Uncontended", uncontendedTimes);
  2010. summariseTimings("Minor", minorTimes);
  2011. summariseTimings("Typical", typicalTimes);
  2012. summariseTimings("Over", contendedTimes);
  2013. }
  2014. void summariseTimings(const char * option, UInt64Array & times)
  2015. {
  2016. printf("%11s 1x: cs(%3" I64F "u) spin(%3" I64F "u) atomic(%3" I64F "u) ratomic(%3" I64F "u) cas(%3" I64F "u) rd(%3" I64F "u) wr(%3" I64F "u) "
  2017. "5x: cs(%3" I64F "u) spin(%3" I64F "u) atomic(%3" I64F "u) ratomic(%3" I64F "u) cas(%3" I64F "u) rd(%3" I64F "u) wr(%3" I64F "u)\n", option,
  2018. times.item(0), times.item(4), times.item(8), times.item(12), times.item(14), times.item(19), times.item(23),
  2019. times.item(2), times.item(6), times.item(10), times.item(13), times.item(15), times.item(21), times.item(25));
  2020. }
  2021. private:
  2022. UInt64Array uncontendedTimes;
  2023. UInt64Array minorTimes;
  2024. UInt64Array typicalTimes;
  2025. UInt64Array contendedTimes;
  2026. };
  2027. CPPUNIT_TEST_SUITE_REGISTRATION(AtomicTimingTest);
  2028. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(AtomicTimingTest, "AtomicTimingTest");
  2029. //=====================================================================================================================
  2030. class MachineInfoTimingTest : public CppUnit::TestFixture
  2031. {
  2032. CPPUNIT_TEST_SUITE(MachineInfoTimingTest);
  2033. CPPUNIT_TEST(runAllTests);
  2034. CPPUNIT_TEST_SUITE_END();
  2035. public:
  2036. void getSystemTiming()
  2037. {
  2038. const unsigned num = 10000;
  2039. CpuInfo temp;
  2040. CCycleTimer timer;
  2041. for (unsigned i=0; i < num; i++)
  2042. temp.getSystemTimes();
  2043. printf("Time to get system cpu activity = %" I64F "uns\n", timer.elapsedNs()/num);
  2044. }
  2045. void getProcessTiming()
  2046. {
  2047. const unsigned num = 10000;
  2048. CpuInfo temp;
  2049. CCycleTimer timer;
  2050. for (unsigned i=0; i < num; i++)
  2051. temp.getProcessTimes();
  2052. printf("Time to get process cpu activity = %" I64F "uns\n", timer.elapsedNs()/num);
  2053. }
  2054. void runAllTests()
  2055. {
  2056. getSystemTiming();
  2057. getSystemTiming(); // Second call seems to be faster - so more representative
  2058. getProcessTiming();
  2059. getProcessTiming(); // Second call seems to be faster - so more representative
  2060. CpuInfo prevSystem;
  2061. CpuInfo prevProcess;
  2062. CpuInfo curProcess(true, false);
  2063. CpuInfo curSystem(false, true);
  2064. volatile unsigned x = 0;
  2065. for (unsigned i=0; i < 10; i++)
  2066. {
  2067. prevProcess = curProcess;
  2068. prevSystem = curSystem;
  2069. curProcess.getProcessTimes();
  2070. curSystem.getSystemTimes();
  2071. CpuInfo deltaProcess = curProcess - prevProcess;
  2072. CpuInfo deltaSystem = curSystem - prevSystem;
  2073. if (deltaSystem.getTotalNs())
  2074. {
  2075. printf(" System: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u) ",
  2076. (unsigned)(deltaSystem.getUserNs() / 1000000), (unsigned)(deltaSystem.getSystemNs() / 1000000), (unsigned)(deltaSystem.getTotalNs() / 1000000),
  2077. (unsigned)((deltaSystem.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaSystem.getNumContextSwitches());
  2078. printf(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)\n",
  2079. (unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000),
  2080. (unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches());
  2081. }
  2082. for (unsigned j=0; j < i*100000000; j++)
  2083. x += j*j;
  2084. Sleep(1000);
  2085. }
  2086. }
  2087. };
  2088. CPPUNIT_TEST_SUITE_REGISTRATION(MachineInfoTimingTest);
  2089. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(MachineInfoTimingTest, "MachineInfoTimingTest");
  2090. class JlibIOTest : public CppUnit::TestFixture
  2091. {
  2092. CPPUNIT_TEST_SUITE(JlibIOTest);
  2093. CPPUNIT_TEST(test);
  2094. CPPUNIT_TEST_SUITE_END();
  2095. public:
  2096. void test()
  2097. {
  2098. unsigned numTestLines = 1000;
  2099. const char *newlines[] = { "\n", "\r\n" };
  2100. for (unsigned pEol=0; pEol<2; pEol++) // twice, once for preserveEols=false, once for preserveEols=true
  2101. {
  2102. for (unsigned nl=0; nl<2; nl++) // twice, once for each type of newline
  2103. {
  2104. const char *testTxt = " : Some random text for test line";
  2105. OwnedIFile iFile = createIFile("JlibIOTest.txt");
  2106. CRC32 writeCrc, readCrc;
  2107. {
  2108. OwnedIFileIO iFileIO = iFile->open(IFOcreate);
  2109. OwnedIFileIOStream stream = createIOStream(iFileIO);
  2110. for (unsigned l=0; l<numTestLines; l++)
  2111. {
  2112. VStringBuffer line("%u%s%s", l+1, testTxt, newlines[nl]);
  2113. stream->write(line.length(), line.str());
  2114. writeCrc.tally(line.length(), line.str());
  2115. }
  2116. }
  2117. {
  2118. OwnedIFileIO iFileIO = iFile->open(IFOread);
  2119. OwnedIFileIOStream stream = createIOStream(iFileIO); // NB: unbuffered
  2120. Owned<IStreamLineReader> lineReader = createLineReader(stream, 0==pEol, strlen(testTxt)); // NB: deliberately make chunkSize small so will end up having to read more
  2121. while (true)
  2122. {
  2123. StringBuffer line;
  2124. if (!lineReader->readLine(line))
  2125. {
  2126. if (pEol==1)
  2127. line.append(newlines[nl]);
  2128. readCrc.tally(line.length(), line.str());
  2129. }
  2130. else
  2131. break;
  2132. }
  2133. }
  2134. CPPUNIT_ASSERT(writeCrc.get() == readCrc.get());
  2135. }
  2136. }
  2137. }
  2138. };
  2139. CPPUNIT_TEST_SUITE_REGISTRATION(JlibIOTest);
  2140. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION(JlibIOTest, "JlibIOTest");
  2141. class JlibCompressionTestsStress : public CppUnit::TestFixture
  2142. {
  2143. CPPUNIT_TEST_SUITE(JlibCompressionTestsStress);
  2144. CPPUNIT_TEST(test);
  2145. CPPUNIT_TEST_SUITE_END();
  2146. public:
  2147. void test()
  2148. {
  2149. try
  2150. {
  2151. size32_t sz = 100*0x100000; // 100MB
  2152. MemoryBuffer src;
  2153. src.ensureCapacity(sz);
  2154. MemoryBuffer compressed;
  2155. const char *aesKey = "012345678901234567890123";
  2156. Owned<ICompressHandlerIterator> iter = getCompressHandlerIterator();
  2157. StringBuffer tmp;
  2158. unsigned card = 0;
  2159. size32_t rowSz = 0;
  2160. while (true)
  2161. {
  2162. size32_t cLen = src.length();
  2163. if (cLen > sz)
  2164. break;
  2165. src.append(cLen);
  2166. tmp.clear().appendf("%10u", cLen);
  2167. src.append(tmp.length(), tmp.str());
  2168. src.append(++card % 52);
  2169. src.append(crc32((const char *)&cLen, sizeof(cLen), 0));
  2170. unsigned ccrc = crc32((const char *)&card, sizeof(card), 0);
  2171. tmp.clear().appendf("%10u", ccrc);
  2172. src.append(tmp.length(), tmp.str());
  2173. tmp.clear().appendf("%20u", (++card % 10));
  2174. src.append(tmp.length(), tmp.str());
  2175. if (0 == rowSz)
  2176. rowSz = src.length();
  2177. else
  2178. {
  2179. dbgassertex(0 == (src.length() % rowSz));
  2180. }
  2181. }
  2182. printf("\nAlgorithm || Compression Time (ms) || Decompression Time (ms) || Compression Ratio\n");
  2183. ForEach(*iter)
  2184. {
  2185. compressed.clear();
  2186. ICompressHandler &handler = iter->query();
  2187. Owned<ICompressor> compressor = handler.getCompressor(streq("AES", handler.queryType()) ? aesKey: nullptr);
  2188. CCycleTimer timer;
  2189. compressor->open(compressed, sz);
  2190. compressor->startblock();
  2191. const byte *ptr = src.bytes();
  2192. const byte *ptrEnd = ptr + src.length();
  2193. while (ptr != ptrEnd)
  2194. {
  2195. compressor->write(ptr, rowSz);
  2196. ptr += rowSz;
  2197. }
  2198. compressor->commitblock();
  2199. compressor->close();
  2200. cycle_t compressCycles = timer.elapsedCycles();
  2201. Owned<IExpander> expander = handler.getExpander(streq("AES", handler.queryType()) ? aesKey: nullptr);
  2202. timer.reset();
  2203. size32_t required = expander->init(compressed.bytes());
  2204. MemoryBuffer tgt(required);
  2205. expander->expand(tgt.bufferBase());
  2206. tgt.setWritePos(required);
  2207. cycle_t decompressCycles = timer.elapsedCycles();
  2208. float ratio = (float)(src.length()) / compressed.length();
  2209. printf("%9s || %21u || %23u || %17.2f [ %u, %u ]\n", handler.queryType(), (unsigned)cycle_to_millisec(compressCycles), (unsigned)cycle_to_millisec(decompressCycles), ratio, src.length(), compressed.length());
  2210. CPPUNIT_ASSERT(tgt.length() >= sz);
  2211. CPPUNIT_ASSERT(0 == memcmp(src.bufferBase(), tgt.bufferBase(), sz));
  2212. }
  2213. }
  2214. catch (IException *e)
  2215. {
  2216. EXCLOG(e, nullptr);
  2217. throw;
  2218. }
  2219. }
  2220. };
  2221. CPPUNIT_TEST_SUITE_REGISTRATION( JlibCompressionTestsStress );
  2222. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibCompressionTestsStress, "JlibCompressionTestsStress" );
  2223. class JlibFriendlySizeTest : public CppUnit::TestFixture
  2224. {
  2225. CPPUNIT_TEST_SUITE(JlibFriendlySizeTest);
  2226. CPPUNIT_TEST(test);
  2227. CPPUNIT_TEST_SUITE_END();
  2228. public:
  2229. void test()
  2230. {
  2231. CPPUNIT_ASSERT(friendlyStringToSize("0") == 0);
  2232. CPPUNIT_ASSERT(friendlyStringToSize("2000") == 2000);
  2233. CPPUNIT_ASSERT(friendlyStringToSize("1K") == 1000);
  2234. CPPUNIT_ASSERT(friendlyStringToSize("2Ki") == 2048);
  2235. CPPUNIT_ASSERT(friendlyStringToSize("1M") == 1000000);
  2236. CPPUNIT_ASSERT(friendlyStringToSize("2Mi") == 2048*1024);
  2237. CPPUNIT_ASSERT(friendlyStringToSize("1G") == 1000000000);
  2238. CPPUNIT_ASSERT(friendlyStringToSize("2Gi") == 2048llu*1024*1024);
  2239. CPPUNIT_ASSERT(friendlyStringToSize("1T") == 1000000000000ll);
  2240. CPPUNIT_ASSERT(friendlyStringToSize("2Ti") == 2048llu*1024*1024*1024);
  2241. CPPUNIT_ASSERT(friendlyStringToSize("1P") == 1000000000000000ll);
  2242. CPPUNIT_ASSERT(friendlyStringToSize("2Pi") == 2048llu*1024*1024*1024*1024);
  2243. CPPUNIT_ASSERT(friendlyStringToSize("1E") == 1000000000000000000ll);
  2244. CPPUNIT_ASSERT(friendlyStringToSize("2Ei") == 2048llu*1024*1024*1024*1024*1024);
  2245. try
  2246. {
  2247. friendlyStringToSize("1Kb");
  2248. CPPUNIT_ASSERT(false);
  2249. }
  2250. catch (IException *E)
  2251. {
  2252. E->Release();
  2253. }
  2254. }
  2255. };
  2256. CPPUNIT_TEST_SUITE_REGISTRATION( JlibFriendlySizeTest );
  2257. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibFriendlySizeTest, "JlibFriendlySizeTest" );
  2258. static stat_type readCheckStatisticValue(const char * cur, StatisticMeasure measure)
  2259. {
  2260. const char * end = nullptr;
  2261. stat_type ret = readStatisticValue(cur, &end, measure);
  2262. CPPUNIT_ASSERT(end && *end == '!');
  2263. return ret;
  2264. }
  2265. class JlibStatsTest : public CppUnit::TestFixture
  2266. {
  2267. CPPUNIT_TEST_SUITE(JlibStatsTest);
  2268. CPPUNIT_TEST(test);
  2269. CPPUNIT_TEST_SUITE_END();
  2270. public:
  2271. void test()
  2272. {
  2273. StringBuffer temp;
  2274. CPPUNIT_ASSERT(readCheckStatisticValue("100s!", SMeasureTimeNs) == U64C(100000000000));
  2275. CPPUNIT_ASSERT(readCheckStatisticValue("100ms!", SMeasureTimeNs) == U64C(100000000));
  2276. CPPUNIT_ASSERT(readCheckStatisticValue("100us!", SMeasureTimeNs) == U64C(100000));
  2277. CPPUNIT_ASSERT(readCheckStatisticValue("100ns!", SMeasureTimeNs) == U64C(100));
  2278. CPPUNIT_ASSERT_EQUAL(U64C(1000000000), readCheckStatisticValue("0:0:1!", SMeasureTimeNs));
  2279. CPPUNIT_ASSERT_EQUAL(U64C(60000000000), readCheckStatisticValue("0:1:0!", SMeasureTimeNs));
  2280. CPPUNIT_ASSERT_EQUAL(U64C(3600000000000), readCheckStatisticValue("1:0:0!", SMeasureTimeNs));
  2281. CPPUNIT_ASSERT_EQUAL(U64C(3600123456789), readCheckStatisticValue("1:0:0.123456789!", SMeasureTimeNs));
  2282. CPPUNIT_ASSERT_EQUAL(U64C(1000), readCheckStatisticValue("0:0:0.000001!", SMeasureTimeNs));
  2283. CPPUNIT_ASSERT_EQUAL(U64C(3600000000000), readCheckStatisticValue("1:0:0!", SMeasureTimeNs));
  2284. CPPUNIT_ASSERT_EQUAL(U64C(86412123456789), readCheckStatisticValue("1d 0:0:12.123456789!", SMeasureTimeNs));
  2285. CPPUNIT_ASSERT_EQUAL(U64C(86460123456789), readCheckStatisticValue("1d 0:1:0.123456789!", SMeasureTimeNs));
  2286. CPPUNIT_ASSERT_EQUAL(U64C(1000), readCheckStatisticValue("1970-01-01T00:00:00.001Z!", SMeasureTimestampUs));
  2287. CPPUNIT_ASSERT_EQUAL(std::string("1970-01-01T00:00:00.001Z"), std::string(formatStatistic(temp.clear(), 1000, SMeasureTimestampUs)));
  2288. CPPUNIT_ASSERT_EQUAL(U64C(1608899696789000), readCheckStatisticValue("2020-12-25T12:34:56.789Z!", SMeasureTimestampUs));
  2289. CPPUNIT_ASSERT_EQUAL(std::string("2020-12-25T12:34:56.789Z"), std::string(formatStatistic(temp.clear(), U64C(1608899696789000), SMeasureTimestampUs)));
  2290. CPPUNIT_ASSERT_EQUAL(U64C(1608899696789000), readCheckStatisticValue("1608899696789000!", SMeasureTimestampUs));
  2291. }
  2292. };
  2293. CPPUNIT_TEST_SUITE_REGISTRATION( JlibStatsTest );
  2294. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibStatsTest, "JlibStatsTest" );
  2295. class HashTableTests : public CppUnit::TestFixture
  2296. {
  2297. CPPUNIT_TEST_SUITE( HashTableTests );
  2298. CPPUNIT_TEST(testTimedCache);
  2299. CPPUNIT_TEST_SUITE_END();
  2300. void testTimedCache()
  2301. {
  2302. unsigned hv = 0;
  2303. unsigned __int64 inputHvSum = 0;
  2304. unsigned __int64 lookupHvSum = 0;
  2305. CTimeLimitedCache<unsigned, unsigned> cache(100); // 100ms timeout
  2306. for (unsigned i=0; i<10; i++)
  2307. {
  2308. hv = hashc((const byte *)&i,sizeof(i), hv);
  2309. inputHvSum += hv;
  2310. cache.add(i, hv);
  2311. unsigned lookupHv;
  2312. CPPUNIT_ASSERT(cache.get(i, lookupHv));
  2313. lookupHvSum += lookupHv;
  2314. }
  2315. CPPUNIT_ASSERT(inputHvSum == lookupHvSum);
  2316. MilliSleep(50);
  2317. CPPUNIT_ASSERT(nullptr != cache.query(0, true)); // touch
  2318. MilliSleep(60);
  2319. // all except 0 that was touched should have expired
  2320. CPPUNIT_ASSERT(nullptr != cache.query(0));
  2321. for (unsigned i=1; i<10; i++)
  2322. CPPUNIT_ASSERT(nullptr == cache.query(i));
  2323. }
  2324. };
  2325. CPPUNIT_TEST_SUITE_REGISTRATION( HashTableTests );
  2326. CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( HashTableTests, "HashTableTests" );
  2327. #endif // _USE_CPPUNIT