ccdmain.cpp 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include <platform.h>
  14. #include <signal.h>
  15. #include <jlib.hpp>
  16. #include <jio.hpp>
  17. #include <jmisc.hpp>
  18. #include <jqueue.tpp>
  19. #include <jsocket.hpp>
  20. #include <jlog.hpp>
  21. #include <jprop.hpp>
  22. #include <jfile.hpp>
  23. #include "jutil.hpp"
  24. #include <build-config.h>
  25. #include "dalienv.hpp"
  26. #include "rmtfile.hpp"
  27. #include "ccd.hpp"
  28. #include "ccdquery.hpp"
  29. #include "ccdstate.hpp"
  30. #include "ccdqueue.ipp"
  31. #include "ccdserver.hpp"
  32. #include "ccdlistener.hpp"
  33. #include "ccdsnmp.hpp"
  34. #include "thorplugin.hpp"
  35. #ifdef _USE_CPPUNIT
  36. #include <cppunit/extensions/TestFactoryRegistry.h>
  37. #include <cppunit/ui/text/TestRunner.h>
  38. #endif
  39. //=================================================================================
  40. bool shuttingDown = false;
  41. unsigned numChannels;
  42. unsigned callbackRetries = 3;
  43. unsigned callbackTimeout = 5000;
  44. unsigned lowTimeout = 10000;
  45. unsigned highTimeout = 2000;
  46. unsigned slaTimeout = 2000;
  47. unsigned numServerThreads = 30;
  48. unsigned numSlaveThreads = 30;
  49. bool prestartSlaveThreads = false;
  50. unsigned numRequestArrayThreads = 5;
  51. unsigned headRegionSize;
  52. unsigned ccdMulticastPort;
  53. bool enableHeartBeat = true;
  54. unsigned parallelLoopFlowLimit = 100;
  55. unsigned perChannelFlowLimit = 10;
  56. time_t startupTime;
  57. unsigned statsExpiryTime = 3600;
  58. unsigned miscDebugTraceLevel = 0; // separate trace settings purely for debugging specific items (i.e. all possible locations to look for files at startup)
  59. unsigned readTimeout = 300;
  60. unsigned indexReadChunkSize = 60000;
  61. unsigned maxBlockSize = 10000000;
  62. unsigned maxLockAttempts = 5;
  63. bool pretendAllOpt = false;
  64. bool traceStartStop = false;
  65. bool traceServerSideCache = false;
  66. bool defaultTimeActivities = true;
  67. bool defaultTraceEnabled = false;
  68. unsigned defaultTraceLimit = 10;
  69. unsigned watchActivityId = 0;
  70. unsigned testSlaveFailure = 0;
  71. unsigned restarts = 0;
  72. bool fieldTranslationEnabled = false;
  73. bool mergeSlaveStatistics = true;
  74. PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
  75. bool runOnce = false;
  76. unsigned udpMulticastBufferSize = 262142;
  77. bool roxieMulticastEnabled = true;
  78. IPropertyTree *topology;
  79. MapStringTo<int> *preferredClusters;
  80. StringBuffer topologyFile;
  81. CriticalSection ccdChannelsCrit;
  82. IPropertyTree* ccdChannels;
  83. StringArray allQuerySetNames;
  84. bool allFilesDynamic;
  85. bool lockSuperFiles;
  86. bool useRemoteResources;
  87. bool checkFileDate;
  88. bool lazyOpen;
  89. bool localSlave;
  90. bool ignoreOrphans;
  91. bool doIbytiDelay = true;
  92. unsigned initIbytiDelay; // In MillSec
  93. unsigned minIbytiDelay; // In MillSec
  94. bool copyResources;
  95. bool enableKeyDiff = true;
  96. bool chunkingHeap = true;
  97. bool logFullQueries;
  98. bool blindLogging = false;
  99. bool debugPermitted = true;
  100. bool checkCompleted = true;
  101. unsigned preabortKeyedJoinsThreshold = 100;
  102. unsigned preabortIndexReadsThreshold = 100;
  103. bool preloadOnceData;
  104. bool reloadRetriesFailed;
  105. bool selfTestMode = false;
  106. unsigned memoryStatsInterval = 0;
  107. memsize_t defaultMemoryLimit;
  108. unsigned defaultTimeLimit[3] = {0, 0, 0};
  109. unsigned defaultWarnTimeLimit[3] = {0, 5000, 5000};
  110. unsigned defaultThorConnectTimeout;
  111. unsigned defaultParallelJoinPreload = 0;
  112. unsigned defaultPrefetchProjectPreload = 10;
  113. unsigned defaultConcatPreload = 0;
  114. unsigned defaultFetchPreload = 0;
  115. unsigned defaultFullKeyedJoinPreload = 0;
  116. unsigned defaultKeyedJoinPreload = 0;
  117. unsigned dafilesrvLookupTimeout = 10000;
  118. bool defaultCheckingHeap = false;
  119. unsigned defaultStrandBlockSize = 512;
  120. unsigned defaultForceNumStrands = 0;
  121. unsigned slaveQueryReleaseDelaySeconds = 60;
  122. unsigned coresPerQuery = 0;
  123. unsigned logQueueLen;
  124. unsigned logQueueDrop;
  125. bool useLogQueue;
  126. bool fastLaneQueue;
  127. unsigned mtu_size = 1400; // upper limit on outbound buffer size - allow some header room too
  128. StringBuffer fileNameServiceDali;
  129. StringBuffer roxieName;
  130. bool trapTooManyActiveQueries;
  131. unsigned maxEmptyLoopIterations;
  132. unsigned maxGraphLoopIterations;
  133. bool probeAllRows;
  134. bool steppingEnabled = true;
  135. bool simpleLocalKeyedJoins = true;
  136. unsigned __int64 minFreeDiskSpace = 1024 * 0x100000; // default to 1 GB
  137. unsigned socketCheckInterval = 5000;
  138. StringBuffer logDirectory;
  139. StringBuffer pluginDirectory;
  140. StringBuffer queryDirectory;
  141. StringBuffer codeDirectory;
  142. StringBuffer tempDirectory;
  143. ClientCertificate clientCert;
  144. bool useHardLink;
  145. unsigned maxFileAge[2] = {0xffffffff, 60*60*1000}; // local files don't expire, remote expire in 1 hour, by default
  146. unsigned minFilesOpen[2] = {2000, 500};
  147. unsigned maxFilesOpen[2] = {4000, 1000};
  148. unsigned myNodeIndex = (unsigned) -1;
  149. SocketEndpoint ownEP;
  150. HardwareInfo hdwInfo;
  151. unsigned parallelAggregate;
  152. bool inMemoryKeysEnabled = true;
  153. unsigned serverSideCacheSize = 0;
  154. bool nodeCachePreload = false;
  155. unsigned nodeCacheMB = 100;
  156. unsigned leafCacheMB = 50;
  157. unsigned blobCacheMB = 0;
  158. unsigned roxiePort = 0;
  159. Owned<IPerfMonHook> perfMonHook;
  160. MODULE_INIT(INIT_PRIORITY_STANDARD)
  161. {
  162. topology = NULL;
  163. ccdChannels = NULL;
  164. return true;
  165. }
  166. MODULE_EXIT()
  167. {
  168. ::Release(topology);
  169. ::Release(ccdChannels);
  170. }
  171. //=========================================================================================
  172. //////////////////////////////////////////////////////////////////////////////////////////////
  173. extern "C" void caughtSIGPIPE(int sig)
  174. {
  175. DBGLOG("Caught sigpipe %d", sig);
  176. }
  177. extern "C" void caughtSIGHUP(int sig)
  178. {
  179. DBGLOG("Caught sighup %d", sig);
  180. }
  181. extern "C" void caughtSIGALRM(int sig)
  182. {
  183. DBGLOG("Caught sigalrm %d", sig);
  184. }
  185. extern "C" void caughtSIGTERM(int sig)
  186. {
  187. DBGLOG("Caught sigterm %d", sig);
  188. }
  189. void init_signals()
  190. {
  191. // signal(SIGTERM, caughtSIGTERM);
  192. #ifndef _WIN32
  193. signal(SIGPIPE, caughtSIGPIPE);
  194. signal(SIGHUP, caughtSIGHUP);
  195. signal(SIGALRM, caughtSIGALRM);
  196. #endif
  197. }
  198. //=========================================================================================
  199. class Waiter : public CInterface, implements IAbortHandler
  200. {
  201. Semaphore aborted;
  202. public:
  203. IMPLEMENT_IINTERFACE;
  204. bool wait(unsigned timeout)
  205. {
  206. return aborted.wait(timeout);
  207. }
  208. void wait()
  209. {
  210. aborted.wait();
  211. }
  212. bool onAbort()
  213. {
  214. aborted.signal();
  215. roxieMetrics.clear();
  216. #ifdef _DEBUG
  217. return false; // we want full leak checking info
  218. #else
  219. return true; // we don't care - just exit as fast as we can
  220. #endif
  221. }
  222. } waiter;
  223. void closedown()
  224. {
  225. Owned<IFile> sentinelFile = createSentinelTarget();
  226. removeSentinelFile(sentinelFile);
  227. waiter.onAbort();
  228. }
  229. void getAccessList(const char *aclName, const IPropertyTree *topology, IPropertyTree *aclInfo)
  230. {
  231. StringBuffer xpath;
  232. xpath.append("ACL[@name='").append(aclName).append("']");
  233. if (aclInfo->queryPropTree(xpath))
  234. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - recursive ACL definition of %s", aclName);
  235. Owned<IPropertyTree> X = createPTree("ACL");
  236. X->setProp("@name", aclName);
  237. aclInfo->addPropTree("ACL", X.getClear());
  238. Owned<IPropertyTree> acl = topology->getPropTree(xpath.str());
  239. if (!acl)
  240. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - acl %s not found", aclName);
  241. Owned<IPropertyTreeIterator> access = acl->getElements("Access");
  242. ForEach(*access)
  243. {
  244. IPropertyTree &child = access->query();
  245. const char *base = child.queryProp("@base");
  246. if (base)
  247. getAccessList(base, topology, aclInfo);
  248. else
  249. aclInfo->addPropTree(child.queryName(), LINK(&child));
  250. }
  251. aclInfo->removeProp(xpath);
  252. }
  253. bool ipMatch(IpAddress &ip)
  254. {
  255. return ip.isLocal();
  256. }
  257. void addSlaveChannel(unsigned channel, unsigned level)
  258. {
  259. StringBuffer xpath;
  260. xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
  261. if (ccdChannels->hasProp(xpath.str()))
  262. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channel %d repeated", channel);
  263. IPropertyTree *ci = createPTree("RoxieSlaveProcess");
  264. ci->setPropInt("@channel", channel);
  265. ci->setPropInt("@subChannel", numSlaves[channel]);
  266. ccdChannels->addPropTree("RoxieSlaveProcess", ci);
  267. }
  268. void addChannel(unsigned nodeNumber, unsigned channel, unsigned level)
  269. {
  270. numSlaves[channel]++;
  271. if (nodeNumber == myNodeIndex && channel > 0)
  272. {
  273. assertex(channel <= numChannels);
  274. assertex(!replicationLevel[channel]);
  275. replicationLevel[channel] = level;
  276. addSlaveChannel(channel, level);
  277. }
  278. if (!localSlave)
  279. {
  280. addEndpoint(channel, getNodeAddress(nodeNumber), ccdMulticastPort);
  281. }
  282. }
  283. extern void doUNIMPLEMENTED(unsigned line, const char *file)
  284. {
  285. throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "UNIMPLEMENTED at %s:%d", file, line);
  286. }
  287. void FatalError(const char *format, ...)
  288. {
  289. va_list args;
  290. va_start(args, format);
  291. StringBuffer errMsg;
  292. errMsg.valist_appendf(format, args);
  293. va_end(args);
  294. Owned<IException> E = MakeStringException(MSGAUD_operator, ROXIE_INTERNAL_ERROR, "%s", errMsg.str());
  295. EXCLOG(E, "Fatal error");
  296. Sleep(5000);
  297. _exit(1);
  298. }
  299. // If changing these, please change roxie.cpp's roxie_server_usage() as well
  300. static void roxie_common_usage(const char * progName)
  301. {
  302. StringBuffer program;
  303. program.append(progName);
  304. getFileNameOnly(program, false);
  305. // Things that are also relevant to stand-alone executables
  306. printf("Usage: %s [options]\n", program.str());
  307. printf("\nOptions:\n");
  308. printf("\t--daliServers=[host1,...]\t: List of Dali servers to use\n");
  309. printf("\t--tracelevel=[integer]\t: Amount of information to dump on logs\n");
  310. printf("\t--stdlog=[boolean]\t: Standard log format (based on tracelevel)\n");
  311. printf("\t--logfile\t: Outputs to logfile, rather than stdout\n");
  312. printf("\t--help|-h\t: This message\n");
  313. printf("\n");
  314. }
  315. class MAbortHandler : implements IExceptionHandler
  316. {
  317. unsigned dummy; // to avoid complaints about an empty class...
  318. public:
  319. MAbortHandler() : dummy(0) {};
  320. virtual bool fireException(IException *e)
  321. {
  322. ForEachItemIn(idx, socketListeners)
  323. {
  324. socketListeners.item(idx).stopListening();
  325. }
  326. return false; // It returns to excsighandler() to abort!
  327. }
  328. } abortHandler;
  329. #ifdef _WIN32
  330. int myhook(int alloctype, void *, size_t nSize, int p1, long allocSeq, const unsigned char *file, int line)
  331. {
  332. // Handy place to put breakpoints when tracking down obscure memory leaks...
  333. if (nSize==68 && !file)
  334. {
  335. DBGLOG("memory hook matched");
  336. }
  337. return true;
  338. }
  339. #endif
  340. void saveTopology()
  341. {
  342. // Write back changes that have been made via certain control:xxx changes, so that they survive a roxie restart
  343. // Note that they are overwritten when Roxie is manually stopped/started via hpcc-init service - these changes
  344. // are only intended to be temporary for the current session
  345. try
  346. {
  347. saveXML(topologyFile.str(), topology);
  348. }
  349. catch (IException *E)
  350. {
  351. // If we can't save the topology, then tough. Carry on without it. Changes will not survive an unexpected roxie restart
  352. EXCLOG(E, "Error saving topology file");
  353. E->Release();
  354. }
  355. }
  356. class CHpccProtocolPluginCtx : public CInterface, implements IHpccProtocolPluginContext
  357. {
  358. public:
  359. IMPLEMENT_IINTERFACE;
  360. virtual int ctxGetPropInt(const char *propName, int defaultValue) const
  361. {
  362. return topology->getPropInt(propName, defaultValue);
  363. }
  364. virtual bool ctxGetPropBool(const char *propName, bool defaultValue) const
  365. {
  366. return topology->getPropBool(propName, defaultValue);
  367. }
  368. virtual const char *ctxQueryProp(const char *propName) const
  369. {
  370. return topology->queryProp(propName);
  371. }
  372. };
  373. int STARTQUERY_API start_query(int argc, const char *argv[])
  374. {
  375. EnableSEHtoExceptionMapping();
  376. setTerminateOnSEH();
  377. init_signals();
  378. // We need to do the above BEFORE we call InitModuleObjects
  379. InitModuleObjects();
  380. getDaliServixPort();
  381. init_signals();
  382. // stand alone usage only, not server
  383. for (unsigned i=0; i<(unsigned)argc; i++)
  384. {
  385. if (stricmp(argv[i], "--help")==0 ||
  386. stricmp(argv[i], "-h")==0)
  387. {
  388. roxie_common_usage(argv[0]);
  389. return EXIT_SUCCESS;
  390. }
  391. }
  392. #ifdef _USE_CPPUNIT
  393. if (argc>=2 && stricmp(argv[1], "-selftest")==0)
  394. {
  395. selfTestMode = true;
  396. queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_prefix);
  397. CppUnit::TextUi::TestRunner runner;
  398. if (argc==2)
  399. {
  400. CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
  401. runner.addTest( registry.makeTest() );
  402. }
  403. else
  404. {
  405. // MORE - maybe add a 'list' function here?
  406. for (int name = 2; name < argc; name++)
  407. {
  408. if (stricmp(argv[name], "-q")==0)
  409. {
  410. traceLevel = 0;
  411. roxiemem::memTraceLevel = 0;
  412. removeLog();
  413. }
  414. else
  415. {
  416. CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry(argv[name]);
  417. runner.addTest( registry.makeTest() );
  418. }
  419. }
  420. }
  421. bool wasSucessful = runner.run( "", false );
  422. releaseAtoms();
  423. return wasSucessful;
  424. }
  425. #endif
  426. #ifdef _DEBUG
  427. #ifdef _WIN32
  428. _CrtSetAllocHook(myhook);
  429. #endif
  430. #endif
  431. #ifndef __64BIT__
  432. // Restrict stack sizes on 32-bit systems
  433. Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
  434. #endif
  435. srand( (unsigned)time( NULL ) );
  436. ccdChannels = createPTree("Channels");
  437. char currentDirectory[_MAX_DIR];
  438. if (!getcwd(currentDirectory, sizeof(currentDirectory)))
  439. throw MakeStringException(ROXIE_INTERNAL_ERROR, "getcwd failed (%d)", errno);
  440. codeDirectory.set(currentDirectory);
  441. addNonEmptyPathSepChar(codeDirectory);
  442. try
  443. {
  444. Owned<IProperties> globals = createProperties(true);
  445. for (int i = 1; i < argc; i++)
  446. globals->loadProp(argv[i], true);
  447. Owned<IFile> sentinelFile = createSentinelTarget();
  448. removeSentinelFile(sentinelFile);
  449. if (globals->hasProp("--topology"))
  450. globals->getProp("--topology", topologyFile);
  451. else
  452. topologyFile.append(codeDirectory).append(PATHSEPCHAR).append("RoxieTopology.xml");
  453. if (checkFileExists(topologyFile.str()))
  454. {
  455. DBGLOG("Loading topology file %s", topologyFile.str());
  456. topology = createPTreeFromXMLFile(topologyFile.str());
  457. saveTopology();
  458. }
  459. else
  460. {
  461. if (globals->hasProp("--topology"))
  462. {
  463. // Explicitly-named topology file SHOULD exist...
  464. throw MakeStringException(ROXIE_INVALID_TOPOLOGY, "topology file %s not found", topologyFile.str());
  465. }
  466. topology=createPTreeFromXMLString(
  467. "<RoxieTopology allFilesDynamic='1' localSlave='1' resolveLocally='1'>"
  468. " <RoxieFarmProcess/>"
  469. " <RoxieServerProcess netAddress='.'/>"
  470. "</RoxieTopology>"
  471. );
  472. int port = globals->getPropInt("--port", 9876);
  473. topology->setPropInt("RoxieFarmProcess/@port", port);
  474. topology->setProp("@daliServers", globals->queryProp("--daliServers"));
  475. topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
  476. topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
  477. }
  478. if (topology->hasProp("PreferredCluster"))
  479. {
  480. preferredClusters = new MapStringTo<int>(true);
  481. Owned<IPropertyTreeIterator> clusters = topology->getElements("PreferredCluster");
  482. ForEach(*clusters)
  483. {
  484. IPropertyTree &child = clusters->query();
  485. const char *name = child.queryProp("@name");
  486. int priority = child.getPropInt("@priority", 100);
  487. if (name && *name)
  488. preferredClusters->setValue(name, priority);
  489. }
  490. }
  491. topology->getProp("@name", roxieName);
  492. if (roxieName.length())
  493. setStatisticsComponentName(SCTroxie, roxieName, true);
  494. else
  495. setStatisticsComponentName(SCTroxie, "roxie", true);
  496. Owned<const IQueryDll> standAloneDll;
  497. if (globals->hasProp("--loadWorkunit"))
  498. {
  499. StringBuffer workunitName;
  500. globals->getProp("--loadWorkunit", workunitName);
  501. standAloneDll.setown(createQueryDll(workunitName));
  502. }
  503. else
  504. {
  505. Owned<ILoadedDllEntry> dll = createExeDllEntry(argv[0]);
  506. if (checkEmbeddedWorkUnitXML(dll))
  507. {
  508. standAloneDll.setown(createExeQueryDll(argv[0]));
  509. runOnce = globals->getPropInt("--port", 0) == 0;
  510. }
  511. }
  512. traceLevel = topology->getPropInt("@traceLevel", runOnce ? 0 : 1);
  513. if (traceLevel > MAXTRACELEVEL)
  514. traceLevel = MAXTRACELEVEL;
  515. udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1);
  516. roxiemem::memTraceLevel = topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1);
  517. soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
  518. miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0);
  519. Linked<IPropertyTree> directoryTree = topology->queryPropTree("Directories");
  520. if (!directoryTree)
  521. {
  522. Owned<IPropertyTree> envFile = getHPCCEnvironment();
  523. if (envFile)
  524. directoryTree.set(envFile->queryPropTree("Software/Directories"));
  525. }
  526. if (directoryTree)
  527. {
  528. getConfigurationDirectory(directoryTree, "query", "roxie", roxieName, queryDirectory);
  529. for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
  530. {
  531. StringBuffer dataDir;
  532. StringBuffer dirId("data");
  533. if (replicationLevel)
  534. dirId.append(replicationLevel+1);
  535. if (getConfigurationDirectory(directoryTree, dirId, "roxie", roxieName, dataDir))
  536. setBaseDirectory(dataDir, replicationLevel, DFD_OSdefault);
  537. }
  538. }
  539. directoryTree.clear();
  540. //Logging stuff
  541. if (globals->getPropBool("--stdlog", traceLevel != 0) || topology->getPropBool("@forceStdLog", false))
  542. queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
  543. else
  544. removeLog();
  545. if (globals->hasProp("--logfile"))
  546. {
  547. Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(topology, "roxie");
  548. lf->setMaxDetail(TopDetail);
  549. lf->beginLogging();
  550. logDirectory.set(lf->queryLogDir());
  551. #ifdef _DEBUG
  552. unsigned useLogQueue = topology->getPropBool("@useLogQueue", false);
  553. #else
  554. unsigned useLogQueue = topology->getPropBool("@useLogQueue", true);
  555. #endif
  556. if (useLogQueue)
  557. {
  558. unsigned logQueueLen = topology->getPropInt("@logQueueLen", 512);
  559. unsigned logQueueDrop = topology->getPropInt("@logQueueDrop", 32);
  560. queryLogMsgManager()->enterQueueingMode();
  561. queryLogMsgManager()->setQueueDroppingLimit(logQueueLen, logQueueDrop);
  562. }
  563. if (globals->getPropBool("--enableSysLog",true))
  564. UseSysLogForOperatorMessages();
  565. }
  566. roxieMetrics.setown(createRoxieMetricsManager());
  567. Owned<IPropertyTreeIterator> userMetrics = topology->getElements("./UserMetric");
  568. ForEach(*userMetrics)
  569. {
  570. IPropertyTree &metric = userMetrics->query();
  571. const char *name = metric.queryProp("@name");
  572. const char *regex= metric.queryProp("@regex");
  573. if (name && regex)
  574. roxieMetrics->addUserMetric(name, regex);
  575. else
  576. throw MakeStringException(ROXIE_INTERNAL_ERROR, "Invalid UserMetric element in topology file - name or regex missing");
  577. }
  578. restarts = globals->getPropInt("--restarts", 0);
  579. const char *preferredSubnet = topology->queryProp("@preferredSubnet");
  580. if (preferredSubnet)
  581. {
  582. const char *preferredSubnetMask = topology->queryProp("@preferredSubnetMask");
  583. if (!preferredSubnetMask) preferredSubnetMask = "255.255.255.0";
  584. if (!setPreferredSubnet(preferredSubnet, preferredSubnetMask))
  585. throw MakeStringException(ROXIE_INTERNAL_ERROR, "Error setting preferred subnet %s mask %s", preferredSubnet, preferredSubnetMask);
  586. }
  587. if (restarts)
  588. {
  589. if (traceLevel)
  590. DBGLOG("Roxie restarting: restarts = %d build = %s", restarts, BUILD_TAG);
  591. setStartRuid(restarts);
  592. }
  593. else
  594. {
  595. if (traceLevel)
  596. {
  597. DBGLOG("Roxie starting, build = %s", BUILD_TAG);
  598. }
  599. }
  600. headRegionSize = topology->getPropInt("@headRegionSize", 50);
  601. ccdMulticastPort = topology->getPropInt("@multicastPort", CCD_MULTICAST_PORT);
  602. statsExpiryTime = topology->getPropInt("@statsExpiryTime", 3600);
  603. roxiemem::memTraceSizeLimit = (memsize_t) topology->getPropInt64("@memTraceSizeLimit", 0);
  604. callbackRetries = topology->getPropInt("@callbackRetries", 3);
  605. callbackTimeout = topology->getPropInt("@callbackTimeout", 5000);
  606. lowTimeout = topology->getPropInt("@lowTimeout", 10000);
  607. highTimeout = topology->getPropInt("@highTimeout", 2000);
  608. slaTimeout = topology->getPropInt("@slaTimeout", 2000);
  609. parallelLoopFlowLimit = topology->getPropInt("@parallelLoopFlowLimit", 100);
  610. perChannelFlowLimit = topology->getPropInt("@perChannelFlowLimit", 10);
  611. copyResources = topology->getPropBool("@copyResources", true);
  612. useRemoteResources = topology->getPropBool("@useRemoteResources", true);
  613. checkFileDate = topology->getPropBool("@checkFileDate", true);
  614. const char *lazyOpenMode = topology->queryProp("@lazyOpen");
  615. if (!lazyOpenMode || stricmp(lazyOpenMode, "smart")==0)
  616. lazyOpen = (restarts > 0);
  617. else
  618. lazyOpen = topology->getPropBool("@lazyOpen", false);
  619. bool useNasTranslation = topology->getPropBool("@useNASTranslation", true);
  620. if (useNasTranslation)
  621. {
  622. Owned<IPropertyTree> nas = envGetNASConfiguration(topology);
  623. envInstallNASHooks(nas);
  624. }
  625. localSlave = topology->getPropBool("@localSlave", false);
  626. doIbytiDelay = topology->getPropBool("@doIbytiDelay", true);
  627. minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
  628. initIbytiDelay = topology->getPropInt("@initIbytiDelay", 50);
  629. allFilesDynamic = topology->getPropBool("@allFilesDynamic", false);
  630. lockSuperFiles = topology->getPropBool("@lockSuperFiles", false);
  631. ignoreOrphans = topology->getPropBool("@ignoreOrphans", true);
  632. chunkingHeap = topology->getPropBool("@chunkingHeap", true);
  633. readTimeout = topology->getPropInt("@readTimeout", 300);
  634. logFullQueries = topology->getPropBool("@logFullQueries", false);
  635. debugPermitted = topology->getPropBool("@debugPermitted", true);
  636. blindLogging = topology->getPropBool("@blindLogging", false);
  637. if (!blindLogging)
  638. logExcessiveSeeks = true;
  639. preloadOnceData = topology->getPropBool("@preloadOnceData", true);
  640. reloadRetriesFailed = topology->getPropBool("@reloadRetriesSuspended", true);
  641. linuxYield = topology->getPropBool("@linuxYield", false);
  642. traceSmartStepping = topology->getPropBool("@traceSmartStepping", false);
  643. useMemoryMappedIndexes = topology->getPropBool("@useMemoryMappedIndexes", false);
  644. traceJHtreeAllocations = topology->getPropBool("@traceJHtreeAllocations", false);
  645. flushJHtreeCacheOnOOM = topology->getPropBool("@flushJHtreeCacheOnOOM", true);
  646. fastLaneQueue = topology->getPropBool("@fastLaneQueue", true);
  647. udpOutQsPriority = topology->getPropInt("@udpOutQsPriority", 0);
  648. udpSnifferEnabled = topology->getPropBool("@udpSnifferEnabled", true);
  649. udpInlineCollation = topology->getPropBool("@udpInlineCollation", false);
  650. udpInlineCollationPacketLimit = topology->getPropInt("@udpInlineCollationPacketLimit", 50);
  651. udpSendCompletedInData = topology->getPropBool("@udpSendCompletedInData", false);
  652. udpRetryBusySenders = topology->getPropInt("@udpRetryBusySenders", 0);
  653. udpMaxRetryTimedoutReqs = topology->getPropInt("@udpMaxRetryTimedoutReqs", 0);
  654. udpRequestToSendTimeout = topology->getPropInt("@udpRequestToSendTimeout", 5);
  655. // MORE: think of a better way/value/check maybe/and/or based on Roxie server timeout
  656. if (udpRequestToSendTimeout == 0)
  657. udpRequestToSendTimeout = 5;
  658. // MORE: might want to check socket buffer sizes against sys max here instead of udp threads ?
  659. udpMulticastBufferSize = topology->getPropInt("@udpMulticastBufferSize", 262142);
  660. udpFlowSocketsSize = topology->getPropInt("@udpFlowSocketsSize", 131072);
  661. udpLocalWriteSocketSize = topology->getPropInt("@udpLocalWriteSocketSize", 1024000);
  662. roxieMulticastEnabled = topology->getPropBool("@roxieMulticastEnabled", true); // enable use of multicast for sending requests to slaves
  663. if (udpSnifferEnabled && !roxieMulticastEnabled)
  664. {
  665. DBGLOG("WARNING: ignoring udpSnifferEnabled setting as multicast not enabled");
  666. udpSnifferEnabled = false;
  667. }
  668. int ttlTmp = topology->getPropInt("@multicastTTL", 1);
  669. if (ttlTmp < 0)
  670. {
  671. multicastTTL = 1;
  672. WARNLOG("multicastTTL value (%d) invalid, must be >=0, resetting to %u", ttlTmp, multicastTTL);
  673. }
  674. else if (ttlTmp > 255)
  675. {
  676. multicastTTL = 255;
  677. WARNLOG("multicastTTL value (%d) invalid, must be <=%u, resetting to maximum", ttlTmp, multicastTTL);
  678. }
  679. else
  680. multicastTTL = ttlTmp;
  681. indexReadChunkSize = topology->getPropInt("@indexReadChunkSize", 60000);
  682. numSlaveThreads = topology->getPropInt("@slaveThreads", 30);
  683. numServerThreads = topology->getPropInt("@serverThreads", 30);
  684. numRequestArrayThreads = topology->getPropInt("@requestArrayThreads", 5);
  685. maxBlockSize = topology->getPropInt("@maxBlockSize", 10000000);
  686. maxLockAttempts = topology->getPropInt("@maxLockAttempts", 5);
  687. enableHeartBeat = topology->getPropBool("@enableHeartBeat", true);
  688. checkCompleted = topology->getPropBool("@checkCompleted", true);
  689. prestartSlaveThreads = topology->getPropBool("@prestartSlaveThreads", false);
  690. preabortKeyedJoinsThreshold = topology->getPropInt("@preabortKeyedJoinsThreshold", 100);
  691. preabortIndexReadsThreshold = topology->getPropInt("@preabortIndexReadsThreshold", 100);
  692. defaultMemoryLimit = (memsize_t) topology->getPropInt64("@defaultMemoryLimit", 0);
  693. defaultTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeLimit", 0);
  694. defaultTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeLimit", 0);
  695. defaultTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeLimit", 0);
  696. defaultWarnTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeWarning", 0);
  697. defaultWarnTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeWarning", 0);
  698. defaultWarnTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeWarning", 0);
  699. defaultThorConnectTimeout = (unsigned) topology->getPropInt64("@defaultThorConnectTimeout", 60);
  700. defaultXmlReadFlags = topology->getPropBool("@defaultStripLeadingWhitespace", true) ? ptr_ignoreWhiteSpace : ptr_none;
  701. defaultParallelJoinPreload = topology->getPropInt("@defaultParallelJoinPreload", 0);
  702. defaultConcatPreload = topology->getPropInt("@defaultConcatPreload", 0);
  703. defaultFetchPreload = topology->getPropInt("@defaultFetchPreload", 0);
  704. defaultFullKeyedJoinPreload = topology->getPropInt("@defaultFullKeyedJoinPreload", 0);
  705. defaultKeyedJoinPreload = topology->getPropInt("@defaultKeyedJoinPreload", 0);
  706. defaultPrefetchProjectPreload = topology->getPropInt("@defaultPrefetchProjectPreload", 10);
  707. defaultStrandBlockSize = topology->getPropInt("@defaultStrandBlockSize", 512);
  708. defaultForceNumStrands = topology->getPropInt("@defaultForceNumStrands", 0);
  709. defaultCheckingHeap = topology->getPropBool("@checkingHeap", false); // NOTE - not in configmgr - too dangerous!
  710. slaveQueryReleaseDelaySeconds = topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60);
  711. coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
  712. diskReadBufferSize = topology->getPropInt("@diskReadBufferSize", 0x10000);
  713. fieldTranslationEnabled = topology->getPropBool("@fieldTranslationEnabled", false);
  714. pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
  715. memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);
  716. roxiemem::setMemoryStatsInterval(memoryStatsInterval);
  717. pingInterval = topology->getPropInt("@pingInterval", 0);
  718. socketCheckInterval = topology->getPropInt("@socketCheckInterval", runOnce ? 0 : 5000);
  719. memsize_t totalMemoryLimit = (memsize_t) topology->getPropInt64("@totalMemoryLimit", 0);
  720. bool allowHugePages = topology->getPropBool("@heapUseHugePages", false);
  721. bool allowTransparentHugePages = topology->getPropBool("@heapUseTransparentHugePages", true);
  722. bool retainMemory = topology->getPropBool("@heapRetainMemory", false);
  723. if (!totalMemoryLimit)
  724. totalMemoryLimit = 1024 * 0x100000; // 1 Gb;
  725. roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, retainMemory, totalMemoryLimit, 0, NULL, NULL);
  726. traceStartStop = topology->getPropBool("@traceStartStop", false);
  727. traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);
  728. defaultTimeActivities = topology->getPropBool("@timeActivities", true);
  729. defaultTraceEnabled = topology->getPropBool("@traceEnabled", false);
  730. defaultTraceLimit = topology->getPropInt("@traceLimit", 10);
  731. clientCert.certificate.set(topology->queryProp("@certificateFileName"));
  732. clientCert.privateKey.set(topology->queryProp("@privateKeyFileName"));
  733. clientCert.passphrase.set(topology->queryProp("@passphrase"));
  734. useHardLink = topology->getPropBool("@useHardLink", false);
  735. maxFileAge[false] = topology->getPropInt("@localFilesExpire", (unsigned) -1);
  736. maxFileAge[true] = topology->getPropInt("@remoteFilesExpire", 60*60*1000);
  737. minFilesOpen[false] = topology->getPropInt("@minLocalFilesOpen", 2000);
  738. minFilesOpen[true] = topology->getPropInt("@minRemoteFilesOpen", 500);
  739. maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 4000);
  740. maxFilesOpen[true] = topology->getPropInt("@maxRemoteFilesOpen", 1000);
  741. dafilesrvLookupTimeout = topology->getPropInt("@dafilesrvLookupTimeout", 10000);
  742. setRemoteFileTimeouts(dafilesrvLookupTimeout, 0);
  743. topology->getProp("@daliServers", fileNameServiceDali);
  744. trapTooManyActiveQueries = topology->getPropBool("@trapTooManyActiveQueries", true);
  745. maxEmptyLoopIterations = topology->getPropInt("@maxEmptyLoopIterations", 1000);
  746. maxGraphLoopIterations = topology->getPropInt("@maxGraphLoopIterations", 1000);
  747. mergeSlaveStatistics = topology->getPropBool("@mergeSlaveStatistics", true);
  748. enableKeyDiff = topology->getPropBool("@enableKeyDiff", true);
  749. // MORE: Get parms from topology after it is populated from Hardware/computer types section in configenv
  750. // Then if does not match and based on desired action in topolgy, either warn, or fatal exit or .... etc
  751. // Also get prim path and sec from topology
  752. #ifdef _WIN32
  753. getHardwareInfo(hdwInfo, "C:", "D:");
  754. #else // linux
  755. getHardwareInfo(hdwInfo, "/c$", "/d$");
  756. #endif
  757. if (traceLevel)
  758. {
  759. DBGLOG("Current Hardware Info: CPUs=%i, speed=%i MHz, Mem=%i MB , primDisk=%i GB, primFree=%i GB, secDisk=%i GB, secFree=%i GB, NIC=%i",
  760. hdwInfo.numCPUs, hdwInfo.CPUSpeed, hdwInfo.totalMemory,
  761. hdwInfo.primDiskSize, hdwInfo.primFreeSize, hdwInfo.secDiskSize, hdwInfo.secFreeSize, hdwInfo.NICSpeed);
  762. }
  763. parallelAggregate = topology->getPropInt("@parallelAggregate", 0);
  764. if (!parallelAggregate)
  765. parallelAggregate = hdwInfo.numCPUs;
  766. if (!parallelAggregate)
  767. parallelAggregate = 1;
  768. simpleLocalKeyedJoins = topology->getPropBool("@simpleLocalKeyedJoins", true);
  769. inMemoryKeysEnabled = topology->getPropBool("@inMemoryKeysEnabled", true);
  770. serverSideCacheSize = topology->getPropInt("@serverSideCacheSize", 0);
  771. setKeyIndexCacheSize((unsigned)-1); // unbound
  772. nodeCachePreload = topology->getPropBool("@nodeCachePreload", false);
  773. setNodeCachePreload(nodeCachePreload);
  774. nodeCacheMB = topology->getPropInt("@nodeCacheMem", 100);
  775. setNodeCacheMem(nodeCacheMB * 0x100000);
  776. leafCacheMB = topology->getPropInt("@leafCacheMem", 50);
  777. setLeafCacheMem(leafCacheMB * 0x100000);
  778. blobCacheMB = topology->getPropInt("@blobCacheMem", 0);
  779. setBlobCacheMem(blobCacheMB * 0x100000);
  780. unsigned __int64 affinity = topology->getPropInt64("@affinity", 0);
  781. updateAffinity(affinity);
  782. minFreeDiskSpace = topology->getPropInt64("@minFreeDiskSpace", (1024 * 0x100000)); // default to 1 GB
  783. if (topology->getPropBool("@jumboFrames", false))
  784. {
  785. mtu_size = 9000; // upper limit on outbound buffer size - allow some header room too
  786. roxiemem::setDataAlignmentSize(0x2000);
  787. }
  788. else
  789. {
  790. mtu_size = 1400; // upper limit on outbound buffer size - allow some header room too
  791. roxiemem::setDataAlignmentSize(0x400);
  792. }
  793. unsigned pinterval = topology->getPropInt("@systemMonitorInterval",1000*60);
  794. perfMonHook.setown(roxiemem::createRoxieMemStatsPerfMonHook()); // Note - we create even if pinterval is 0, as can be enabled via control message
  795. if (pinterval)
  796. startPerformanceMonitor(pinterval, PerfMonStandard, perfMonHook);
  797. topology->getProp("@pluginDirectory", pluginDirectory);
  798. if (pluginDirectory.length() == 0)
  799. pluginDirectory.append(codeDirectory).append("plugins");
  800. if (queryDirectory.length() == 0)
  801. {
  802. topology->getProp("@queryDir", queryDirectory);
  803. if (queryDirectory.length() == 0)
  804. queryDirectory.append(codeDirectory).append("queries");
  805. }
  806. addNonEmptyPathSepChar(queryDirectory);
  807. queryFileCache().start();
  808. getTempFilePath(tempDirectory, "roxie", topology);
  809. #ifdef _WIN32
  810. topology->addPropBool("@linuxOS", false);
  811. #else
  812. topology->addPropBool("@linuxOS", true);
  813. #endif
  814. allQuerySetNames.appendListUniq(topology->queryProp("@querySets"), ",");
  815. Owned<IPropertyTreeIterator> roxieServers = topology->getElements("./RoxieServerProcess");
  816. ForEach(*roxieServers)
  817. {
  818. IPropertyTree &roxieServer = roxieServers->query();
  819. const char *iptext = roxieServer.queryProp("@netAddress");
  820. unsigned nodeIndex = addRoxieNode(iptext);
  821. if (getNodeAddress(nodeIndex).isLocal())
  822. myNodeIndex = nodeIndex;
  823. if (traceLevel > 3)
  824. DBGLOG("Roxie server %u is at %s", nodeIndex, iptext);
  825. }
  826. if (myNodeIndex == -1)
  827. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - current node is not in server list");
  828. // Set multicast base addresses - must be done before generating slave channels
  829. if (roxieMulticastEnabled && !localSlave)
  830. {
  831. if (topology->queryProp("@multicastBase"))
  832. multicastBase.ipset(topology->queryProp("@multicastBase"));
  833. else
  834. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastBase not set");
  835. if (topology->queryProp("@multicastLast"))
  836. multicastLast.ipset(topology->queryProp("@multicastLast"));
  837. else
  838. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastLast not set");
  839. }
  840. // Generate the slave channels
  841. unsigned numDataCopies = topology->getPropInt("@numDataCopies", 1);
  842. if (!numDataCopies)
  843. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numDataCopies should be > 0");
  844. unsigned channelsPerNode = topology->getPropInt("@channelsPerNode", 1);
  845. unsigned numNodes = getNumNodes();
  846. const char *slaveConfig = topology->queryProp("@slaveConfig");
  847. if (!slaveConfig)
  848. slaveConfig = "simple";
  849. if (strnicmp(slaveConfig, "cyclic", 6) == 0)
  850. {
  851. numChannels = numNodes;
  852. unsigned cyclicOffset = topology->getPropInt("@cyclicOffset", 1);
  853. for (unsigned i=0; i<numNodes; i++)
  854. {
  855. // Note this code is a little confusing - easy to get the cyclic offset backwards
  856. // cyclic offset means node n+offset has copy 2 for channel n, so node n has copy 2 for channel n-offset
  857. int channel = (int)i+1;
  858. for (unsigned copy=0; copy<numDataCopies; copy++)
  859. {
  860. if (channel < 1)
  861. channel = channel + numNodes;
  862. addChannel(i, channel, copy);
  863. channel -= cyclicOffset;
  864. }
  865. }
  866. }
  867. else if (strnicmp(slaveConfig, "overloaded", 10) == 0)
  868. {
  869. if (!channelsPerNode)
  870. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channelsPerNode should be > 0");
  871. numChannels = numNodes * channelsPerNode;
  872. for (unsigned i=0; i<numNodes; i++)
  873. {
  874. int channel = (int)(i+1);
  875. for (unsigned copy=0; copy<channelsPerNode; copy++)
  876. {
  877. addChannel(i, channel, copy);
  878. channel += numNodes;
  879. }
  880. }
  881. }
  882. else // 'Full redundancy' or 'simple' mode
  883. {
  884. if (numNodes % numDataCopies)
  885. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not an integer");
  886. numChannels = numNodes / numDataCopies;
  887. int channel = 1;
  888. for (unsigned i=0; i<numNodes; i++)
  889. {
  890. addChannel(i, channel, 0);
  891. channel++;
  892. if ((unsigned)channel > numChannels)
  893. channel = 1;
  894. }
  895. }
  896. if (!numChannels)
  897. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels calculated at 0");
  898. if (numChannels > 1 && localSlave)
  899. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - localSlave requires single channel (%d channels specified)", numChannels);
  900. // Now we know all the channels, we can open and subscribe the multicast channels
  901. if (!localSlave)
  902. openMulticastSocket();
  903. setDaliServixSocketCaching(true); // enable daliservix caching
  904. loadPlugins();
  905. createDelayedReleaser();
  906. globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
  907. globalPackageSetManager->load();
  908. unsigned snifferChannel = numChannels+2; // MORE - why +2 not +1 ??
  909. ROQ = createOutputQueueManager(snifferChannel, numSlaveThreads);
  910. ROQ->setHeadRegionSize(headRegionSize);
  911. ROQ->start();
  912. Owned<IPacketDiscarder> packetDiscarder = createPacketDiscarder();
  913. #if defined(WIN32) && defined(_DEBUG) && defined(_DEBUG_HEAP_FULL)
  914. int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
  915. tmpFlag |= _CRTDBG_CHECK_ALWAYS_DF;
  916. _CrtSetDbgFlag( tmpFlag );
  917. #endif
  918. EnableSEHtoExceptionMapping();
  919. setSEHtoExceptionHandler(&abortHandler);
  920. Owned<IHpccProtocolPluginContext> protocolCtx = new CHpccProtocolPluginCtx();
  921. if (runOnce)
  922. {
  923. if (globals->hasProp("-wu"))
  924. {
  925. Owned<IHpccProtocolListener> roxieServer = createRoxieWorkUnitListener(1, false);
  926. try
  927. {
  928. VStringBuffer x("-%s", argv[0]);
  929. roxieServer->runOnce(x);
  930. fflush(stdout); // in windows if output is redirected results don't appear without flushing
  931. }
  932. catch (IException *E)
  933. {
  934. EXCLOG(E);
  935. E->Release();
  936. }
  937. }
  938. else
  939. {
  940. Owned<IHpccProtocolPlugin> protocolPlugin = loadHpccProtocolPlugin(protocolCtx, NULL);
  941. Owned<IHpccProtocolListener> roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(getNodeAddress(myNodeIndex), 0, 1, false), 0, 0, NULL);
  942. try
  943. {
  944. const char *format = globals->queryProp("format");
  945. if (!format)
  946. {
  947. if (globals->hasProp("-xml"))
  948. format = "xml";
  949. else if (globals->hasProp("-csv"))
  950. format = "csv";
  951. else if (globals->hasProp("-raw"))
  952. format = "raw";
  953. else
  954. format = "ascii";
  955. }
  956. StringBuffer query;
  957. query.appendf("<roxie format='%s'/>", format);
  958. roxieServer->runOnce(query.str()); // MORE - should use the wu listener instead I suspect
  959. fflush(stdout); // in windows if output is redirected results don't appear without flushing
  960. }
  961. catch (IException *E)
  962. {
  963. EXCLOG(E);
  964. E->Release();
  965. }
  966. }
  967. }
  968. else
  969. {
  970. Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./RoxieFarmProcess");
  971. ForEach(*roxieFarms)
  972. {
  973. IPropertyTree &roxieFarm = roxieFarms->query();
  974. unsigned listenQueue = roxieFarm.getPropInt("@listenQueue", DEFAULT_LISTEN_QUEUE_SIZE);
  975. unsigned numThreads = roxieFarm.getPropInt("@numThreads", numServerThreads);
  976. unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
  977. unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
  978. const IpAddress &ip = getNodeAddress(myNodeIndex);
  979. if (!roxiePort)
  980. {
  981. roxiePort = port;
  982. ownEP.set(roxiePort, ip);
  983. }
  984. bool suspended = roxieFarm.getPropBool("@suspended", false);
  985. Owned <IHpccProtocolListener> roxieServer;
  986. if (port)
  987. {
  988. const char *protocol = roxieFarm.queryProp("@protocol");
  989. const char *soname = roxieFarm.queryProp("@so");
  990. const char *config = roxieFarm.queryProp("@config");
  991. Owned<IHpccProtocolPlugin> protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname);
  992. roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config));
  993. }
  994. else
  995. roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended));
  996. IHpccProtocolMsgSink *sink = roxieServer->queryMsgSink();
  997. const char *aclName = roxieFarm.queryProp("@aclName");
  998. if (aclName && *aclName)
  999. {
  1000. Owned<IPropertyTree> aclInfo = createPTree("AccessInfo");
  1001. getAccessList(aclName, topology, aclInfo);
  1002. Owned<IPropertyTreeIterator> accesses = aclInfo->getElements("Access");
  1003. ForEach(*accesses)
  1004. {
  1005. IPropertyTree &access = accesses->query();
  1006. try
  1007. {
  1008. sink->addAccess(access.getPropBool("@allow", true), access.getPropBool("@allowBlind", true), access.queryProp("@ip"), access.queryProp("@mask"), access.queryProp("@query"), access.queryProp("@error"), access.getPropInt("@errorCode"));
  1009. }
  1010. catch (IException *E)
  1011. {
  1012. StringBuffer s, x;
  1013. E->errorMessage(s);
  1014. E->Release();
  1015. toXML(&access, x, 0, 0);
  1016. throw MakeStringException(ROXIE_ACL_ERROR, "Error in access statement %s: %s", x.str(), s.str());
  1017. }
  1018. }
  1019. }
  1020. socketListeners.append(*roxieServer.getLink());
  1021. time(&startupTime);
  1022. roxieServer->start();
  1023. }
  1024. writeSentinelFile(sentinelFile);
  1025. DBGLOG("Waiting for queries");
  1026. if (pingInterval)
  1027. startPingTimer();
  1028. LocalIAbortHandler abortHandler(waiter);
  1029. waiter.wait();
  1030. }
  1031. shuttingDown = true;
  1032. if (pingInterval)
  1033. stopPingTimer();
  1034. setSEHtoExceptionHandler(NULL);
  1035. while (socketListeners.isItem(0))
  1036. {
  1037. socketListeners.item(0).stop(1000);
  1038. socketListeners.remove(0);
  1039. }
  1040. packetDiscarder->stop();
  1041. packetDiscarder.clear();
  1042. ROQ->stop();
  1043. ROQ->join();
  1044. ROQ->Release();
  1045. ROQ = NULL;
  1046. }
  1047. catch (IException *E)
  1048. {
  1049. StringBuffer x;
  1050. DBGLOG("EXCEPTION: (%d): %s", E->errorCode(), E->errorMessage(x).str());
  1051. E->Release();
  1052. }
  1053. roxieMetrics.clear();
  1054. stopPerformanceMonitor();
  1055. ::Release(globalPackageSetManager);
  1056. globalPackageSetManager = NULL;
  1057. stopDelayedReleaser();
  1058. cleanupPlugins();
  1059. closeMulticastSockets();
  1060. releaseSlaveDynamicFileCache();
  1061. releaseRoxieStateCache();
  1062. setDaliServixSocketCaching(false); // make sure it cleans up or you get bogus memleak reports
  1063. setNodeCaching(false); // ditto
  1064. perfMonHook.clear();
  1065. strdup("Make sure leak checking is working");
  1066. UseSysLogForOperatorMessages(false);
  1067. ExitModuleObjects();
  1068. releaseAtoms();
  1069. strdup("Make sure leak checking is working");
  1070. #ifdef _WIN32
  1071. #ifdef _DEBUG
  1072. #if 1
  1073. StringBuffer leakFileDir(logDirectory.str());
  1074. leakFileDir.append("roxieleaks.log");
  1075. HANDLE h = CreateFile(leakFileDir.str(), GENERIC_READ|GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, 0, 0);
  1076. _CrtSetReportMode( _CRT_WARN, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1077. _CrtSetReportFile( _CRT_WARN, h);
  1078. _CrtSetReportMode( _CRT_ERROR, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1079. _CrtSetReportFile( _CRT_ERROR, h);
  1080. _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1081. _CrtSetReportFile( _CRT_ASSERT, h);
  1082. // _CrtDumpMemoryLeaks(); if you uncomment these lines you get to see the leaks sooner (so can look in debugger at full memory)
  1083. // CloseHandle(h); but there will be additional leaks reported that are not really leaks
  1084. #endif
  1085. #endif
  1086. #endif
  1087. return 0;
  1088. }