ccdmain.cpp 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include <platform.h>
  14. #include <signal.h>
  15. #include <jlib.hpp>
  16. #include <jio.hpp>
  17. #include <jmisc.hpp>
  18. #include <jqueue.tpp>
  19. #include <jsocket.hpp>
  20. #include <jlog.hpp>
  21. #include <jprop.hpp>
  22. #include <jfile.hpp>
  23. #include <jencrypt.hpp>
  24. #include "jutil.hpp"
  25. #include <build-config.h>
  26. #include "rtlformat.hpp"
  27. #include "dalienv.hpp"
  28. #include "rmtfile.hpp"
  29. #include "ccd.hpp"
  30. #include "ccdquery.hpp"
  31. #include "ccdstate.hpp"
  32. #include "ccdqueue.ipp"
  33. #include "ccdserver.hpp"
  34. #include "ccdlistener.hpp"
  35. #include "ccdsnmp.hpp"
  36. #include "thorplugin.hpp"
  37. #if defined (__linux__)
  38. #include <sys/syscall.h>
  39. #include "ioprio.h"
  40. #endif
  41. #ifdef _USE_CPPUNIT
  42. #include <cppunit/extensions/TestFactoryRegistry.h>
  43. #include <cppunit/ui/text/TestRunner.h>
  44. #endif
  45. //=================================================================================
  46. bool shuttingDown = false;
  47. unsigned numChannels;
  48. unsigned callbackRetries = 3;
  49. unsigned callbackTimeout = 5000;
  50. unsigned lowTimeout = 10000;
  51. unsigned highTimeout = 2000;
  52. unsigned slaTimeout = 2000;
  53. unsigned numServerThreads = 30;
  54. unsigned numSlaveThreads = 30;
  55. bool prestartSlaveThreads = false;
  56. unsigned numRequestArrayThreads = 5;
  57. unsigned headRegionSize;
  58. unsigned ccdMulticastPort;
  59. bool enableHeartBeat = true;
  60. unsigned parallelLoopFlowLimit = 100;
  61. unsigned perChannelFlowLimit = 10;
  62. time_t startupTime;
  63. unsigned statsExpiryTime = 3600;
  64. unsigned miscDebugTraceLevel = 0; // separate trace settings purely for debugging specific items (i.e. all possible locations to look for files at startup)
  65. unsigned readTimeout = 300;
  66. unsigned indexReadChunkSize = 60000;
  67. unsigned maxBlockSize = 10000000;
  68. unsigned maxLockAttempts = 5;
  69. bool pretendAllOpt = false;
  70. bool traceStartStop = false;
  71. bool traceServerSideCache = false;
  72. bool defaultTimeActivities = true;
  73. bool defaultTraceEnabled = false;
  74. unsigned defaultTraceLimit = 10;
  75. unsigned watchActivityId = 0;
  76. unsigned testSlaveFailure = 0;
  77. unsigned restarts = 0;
  78. RecordTranslationMode fieldTranslationEnabled = RecordTranslationMode::None;
  79. bool mergeSlaveStatistics = true;
  80. PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
  81. bool runOnce = false;
  82. unsigned udpMulticastBufferSize = 262142;
  83. bool roxieMulticastEnabled = true;
  84. IPropertyTree *topology;
  85. MapStringTo<int> *preferredClusters;
  86. StringBuffer topologyFile;
  87. CriticalSection ccdChannelsCrit;
  88. IPropertyTree* ccdChannels;
  89. StringArray allQuerySetNames;
  90. bool allFilesDynamic;
  91. bool lockSuperFiles;
  92. bool useRemoteResources;
  93. bool checkFileDate;
  94. bool lazyOpen;
  95. bool localSlave;
  96. bool ignoreOrphans;
  97. bool doIbytiDelay = true;
  98. unsigned initIbytiDelay; // In MillSec
  99. unsigned minIbytiDelay; // In MillSec
  100. bool copyResources;
  101. bool enableKeyDiff = true;
  102. bool chunkingHeap = true;
  103. bool logFullQueries;
  104. bool blindLogging = false;
  105. bool debugPermitted = true;
  106. bool checkCompleted = true;
  107. unsigned preabortKeyedJoinsThreshold = 100;
  108. unsigned preabortIndexReadsThreshold = 100;
  109. bool preloadOnceData;
  110. bool reloadRetriesFailed;
  111. bool selfTestMode = false;
  112. int backgroundCopyClass = 0;
  113. int backgroundCopyPrio = 0;
  114. unsigned memoryStatsInterval = 0;
  115. memsize_t defaultMemoryLimit;
  116. unsigned defaultTimeLimit[3] = {0, 0, 0};
  117. unsigned defaultWarnTimeLimit[3] = {0, 5000, 5000};
  118. unsigned defaultThorConnectTimeout;
  119. unsigned defaultParallelJoinPreload = 0;
  120. unsigned defaultPrefetchProjectPreload = 10;
  121. unsigned defaultConcatPreload = 0;
  122. unsigned defaultFetchPreload = 0;
  123. unsigned defaultFullKeyedJoinPreload = 0;
  124. unsigned defaultKeyedJoinPreload = 0;
  125. unsigned dafilesrvLookupTimeout = 10000;
  126. bool defaultCheckingHeap = false;
  127. bool defaultDisableLocalOptimizations = false;
  128. unsigned defaultStrandBlockSize = 512;
  129. unsigned defaultForceNumStrands = 0;
  130. unsigned defaultHeapFlags = roxiemem::RHFnofragment;
  131. unsigned slaveQueryReleaseDelaySeconds = 60;
  132. unsigned coresPerQuery = 0;
  133. unsigned logQueueLen;
  134. unsigned logQueueDrop;
  135. bool useLogQueue;
  136. bool fastLaneQueue;
  137. unsigned mtu_size = 1400; // upper limit on outbound buffer size - allow some header room too
  138. StringBuffer fileNameServiceDali;
  139. StringBuffer roxieName;
  140. bool trapTooManyActiveQueries;
  141. unsigned maxEmptyLoopIterations;
  142. unsigned maxGraphLoopIterations;
  143. bool probeAllRows;
  144. bool steppingEnabled = true;
  145. bool simpleLocalKeyedJoins = true;
  146. unsigned __int64 minFreeDiskSpace = 1024 * 0x100000; // default to 1 GB
  147. unsigned socketCheckInterval = 5000;
  148. StringBuffer logDirectory;
  149. StringBuffer pluginDirectory;
  150. StringBuffer queryDirectory;
  151. StringBuffer codeDirectory;
  152. StringBuffer tempDirectory;
  153. ClientCertificate clientCert;
  154. bool useHardLink;
  155. unsigned maxFileAge[2] = {0xffffffff, 60*60*1000}; // local files don't expire, remote expire in 1 hour, by default
  156. unsigned minFilesOpen[2] = {2000, 500};
  157. unsigned maxFilesOpen[2] = {4000, 1000};
  158. unsigned myNodeIndex = (unsigned) -1;
  159. SocketEndpoint ownEP;
  160. HardwareInfo hdwInfo;
  161. unsigned parallelAggregate;
  162. bool inMemoryKeysEnabled = true;
  163. unsigned serverSideCacheSize = 0;
  164. bool nodeCachePreload = false;
  165. unsigned nodeCacheMB = 100;
  166. unsigned leafCacheMB = 50;
  167. unsigned blobCacheMB = 0;
  168. unsigned roxiePort = 0;
  169. Owned<IPerfMonHook> perfMonHook;
  170. MODULE_INIT(INIT_PRIORITY_STANDARD)
  171. {
  172. topology = NULL;
  173. ccdChannels = NULL;
  174. return true;
  175. }
  176. MODULE_EXIT()
  177. {
  178. ::Release(topology);
  179. ::Release(ccdChannels);
  180. }
  181. //=========================================================================================
  182. //////////////////////////////////////////////////////////////////////////////////////////////
  183. extern "C" void caughtSIGPIPE(int sig)
  184. {
  185. DBGLOG("Caught sigpipe %d", sig);
  186. }
  187. extern "C" void caughtSIGHUP(int sig)
  188. {
  189. DBGLOG("Caught sighup %d", sig);
  190. }
  191. extern "C" void caughtSIGALRM(int sig)
  192. {
  193. DBGLOG("Caught sigalrm %d", sig);
  194. }
  195. extern "C" void caughtSIGTERM(int sig)
  196. {
  197. DBGLOG("Caught sigterm %d", sig);
  198. }
  199. void init_signals()
  200. {
  201. // signal(SIGTERM, caughtSIGTERM);
  202. #ifndef _WIN32
  203. signal(SIGPIPE, caughtSIGPIPE);
  204. signal(SIGHUP, caughtSIGHUP);
  205. signal(SIGALRM, caughtSIGALRM);
  206. #endif
  207. }
  208. //=========================================================================================
  209. class Waiter : public CInterface, implements IAbortHandler
  210. {
  211. Semaphore aborted;
  212. public:
  213. IMPLEMENT_IINTERFACE;
  214. bool wait(unsigned timeout)
  215. {
  216. return aborted.wait(timeout);
  217. }
  218. void wait()
  219. {
  220. aborted.wait();
  221. }
  222. bool onAbort()
  223. {
  224. aborted.signal();
  225. roxieMetrics.clear();
  226. #ifdef _DEBUG
  227. return false; // we want full leak checking info
  228. #else
  229. return true; // we don't care - just exit as fast as we can
  230. #endif
  231. }
  232. } waiter;
  233. void closedown()
  234. {
  235. Owned<IFile> sentinelFile = createSentinelTarget();
  236. removeSentinelFile(sentinelFile);
  237. waiter.onAbort();
  238. }
  239. void getAccessList(const char *aclName, const IPropertyTree *topology, IPropertyTree *aclInfo)
  240. {
  241. StringBuffer xpath;
  242. xpath.append("ACL[@name='").append(aclName).append("']");
  243. if (aclInfo->queryPropTree(xpath))
  244. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - recursive ACL definition of %s", aclName);
  245. aclInfo->addPropTree("ACL")->setProp("@name", aclName);
  246. Owned<IPropertyTree> acl = topology->getPropTree(xpath.str());
  247. if (!acl)
  248. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - acl %s not found", aclName);
  249. Owned<IPropertyTreeIterator> access = acl->getElements("Access");
  250. ForEach(*access)
  251. {
  252. IPropertyTree &child = access->query();
  253. const char *base = child.queryProp("@base");
  254. if (base)
  255. getAccessList(base, topology, aclInfo);
  256. else
  257. aclInfo->addPropTree(child.queryName(), LINK(&child));
  258. }
  259. aclInfo->removeProp(xpath);
  260. }
  261. bool ipMatch(IpAddress &ip)
  262. {
  263. return ip.isLocal();
  264. }
  265. void addSlaveChannel(unsigned channel, unsigned level)
  266. {
  267. StringBuffer xpath;
  268. xpath.appendf("RoxieSlaveProcess[@channel=\"%d\"]", channel);
  269. if (ccdChannels->hasProp(xpath.str()))
  270. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channel %d repeated", channel);
  271. IPropertyTree *ci = ccdChannels->addPropTree("RoxieSlaveProcess");
  272. ci->setPropInt("@channel", channel);
  273. ci->setPropInt("@subChannel", numSlaves[channel]);
  274. }
  275. void addChannel(unsigned nodeNumber, unsigned channel, unsigned level)
  276. {
  277. numSlaves[channel]++;
  278. if (nodeNumber == myNodeIndex && channel > 0)
  279. {
  280. assertex(channel <= numChannels);
  281. assertex(!replicationLevel[channel]);
  282. replicationLevel[channel] = level;
  283. addSlaveChannel(channel, level);
  284. }
  285. if (!localSlave)
  286. {
  287. addEndpoint(channel, getNodeAddress(nodeNumber), ccdMulticastPort);
  288. }
  289. }
  290. extern void doUNIMPLEMENTED(unsigned line, const char *file)
  291. {
  292. throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "UNIMPLEMENTED at %s:%d", sanitizeSourceFile(file), line);
  293. }
  294. void FatalError(const char *format, ...)
  295. {
  296. va_list args;
  297. va_start(args, format);
  298. StringBuffer errMsg;
  299. errMsg.valist_appendf(format, args);
  300. va_end(args);
  301. Owned<IException> E = MakeStringException(MSGAUD_operator, ROXIE_INTERNAL_ERROR, "%s", errMsg.str());
  302. EXCLOG(E, "Fatal error");
  303. Sleep(5000);
  304. _exit(1);
  305. }
  306. // If changing these, please change roxie.cpp's roxie_server_usage() as well
  307. static void roxie_common_usage(const char * progName)
  308. {
  309. StringBuffer program;
  310. program.append(progName);
  311. getFileNameOnly(program, false);
  312. // Things that are also relevant to stand-alone executables
  313. printf("Usage: %s [options]\n", program.str());
  314. printf("\nOptions:\n");
  315. printf("\t--daliServers=[host1,...]\t: List of Dali servers to use\n");
  316. printf("\t--tracelevel=[integer]\t: Amount of information to dump on logs\n");
  317. printf("\t--stdlog=[boolean]\t: Standard log format (based on tracelevel)\n");
  318. printf("\t--logfile\t: Outputs to logfile, rather than stdout\n");
  319. printf("\t--help|-h\t: This message\n");
  320. printf("\n");
  321. }
  322. class MAbortHandler : implements IExceptionHandler
  323. {
  324. unsigned dummy; // to avoid complaints about an empty class...
  325. public:
  326. MAbortHandler() : dummy(0) {};
  327. virtual bool fireException(IException *e)
  328. {
  329. ForEachItemIn(idx, socketListeners)
  330. {
  331. socketListeners.item(idx).stopListening();
  332. }
  333. return false; // It returns to excsighandler() to abort!
  334. }
  335. } abortHandler;
  336. #ifdef _WIN32
  337. int myhook(int alloctype, void *, size_t nSize, int p1, long allocSeq, const unsigned char *file, int line)
  338. {
  339. // Handy place to put breakpoints when tracking down obscure memory leaks...
  340. if (nSize==68 && !file)
  341. {
  342. DBGLOG("memory hook matched");
  343. }
  344. return true;
  345. }
  346. #endif
  347. void saveTopology()
  348. {
  349. // Write back changes that have been made via certain control:xxx changes, so that they survive a roxie restart
  350. // Note that they are overwritten when Roxie is manually stopped/started via hpcc-init service - these changes
  351. // are only intended to be temporary for the current session
  352. try
  353. {
  354. saveXML(topologyFile.str(), topology);
  355. }
  356. catch (IException *E)
  357. {
  358. // If we can't save the topology, then tough. Carry on without it. Changes will not survive an unexpected roxie restart
  359. EXCLOG(E, "Error saving topology file");
  360. E->Release();
  361. }
  362. }
  363. class CHpccProtocolPluginCtx : implements IHpccProtocolPluginContext, public CInterface
  364. {
  365. public:
  366. IMPLEMENT_IINTERFACE;
  367. virtual int ctxGetPropInt(const char *propName, int defaultValue) const
  368. {
  369. return topology->getPropInt(propName, defaultValue);
  370. }
  371. virtual bool ctxGetPropBool(const char *propName, bool defaultValue) const
  372. {
  373. return topology->getPropBool(propName, defaultValue);
  374. }
  375. virtual const char *ctxQueryProp(const char *propName) const
  376. {
  377. return topology->queryProp(propName);
  378. }
  379. };
  380. int STARTQUERY_API start_query(int argc, const char *argv[])
  381. {
  382. EnableSEHtoExceptionMapping();
  383. setTerminateOnSEH();
  384. init_signals();
  385. // We need to do the above BEFORE we call InitModuleObjects
  386. InitModuleObjects();
  387. init_signals();
  388. // stand alone usage only, not server
  389. for (unsigned i=0; i<(unsigned)argc; i++)
  390. {
  391. if (stricmp(argv[i], "--help")==0 ||
  392. stricmp(argv[i], "-h")==0)
  393. {
  394. roxie_common_usage(argv[0]);
  395. return EXIT_SUCCESS;
  396. }
  397. }
  398. #ifdef _USE_CPPUNIT
  399. if (argc>=2 && stricmp(argv[1], "-selftest")==0)
  400. {
  401. selfTestMode = true;
  402. queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_prefix);
  403. CppUnit::TextUi::TestRunner runner;
  404. if (argc==2)
  405. {
  406. CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry();
  407. runner.addTest( registry.makeTest() );
  408. }
  409. else
  410. {
  411. // MORE - maybe add a 'list' function here?
  412. for (int name = 2; name < argc; name++)
  413. {
  414. if (stricmp(argv[name], "-q")==0)
  415. {
  416. traceLevel = 0;
  417. roxiemem::setMemTraceLevel(0);
  418. removeLog();
  419. }
  420. else
  421. {
  422. CppUnit::TestFactoryRegistry &registry = CppUnit::TestFactoryRegistry::getRegistry(argv[name]);
  423. runner.addTest( registry.makeTest() );
  424. }
  425. }
  426. }
  427. bool wasSucessful = runner.run( "", false );
  428. releaseAtoms();
  429. return wasSucessful;
  430. }
  431. #endif
  432. #ifdef _DEBUG
  433. #ifdef _WIN32
  434. _CrtSetAllocHook(myhook);
  435. #endif
  436. #endif
  437. #ifndef __64BIT__
  438. // Restrict stack sizes on 32-bit systems
  439. Thread::setDefaultStackSize(0x10000); // NB under windows requires linker setting (/stack:)
  440. #endif
  441. srand( (unsigned)time( NULL ) );
  442. ccdChannels = createPTree("Channels", ipt_lowmem);
  443. char currentDirectory[_MAX_DIR];
  444. if (!getcwd(currentDirectory, sizeof(currentDirectory)))
  445. throw MakeStringException(ROXIE_INTERNAL_ERROR, "getcwd failed (%d)", errno);
  446. codeDirectory.set(currentDirectory);
  447. addNonEmptyPathSepChar(codeDirectory);
  448. try
  449. {
  450. Owned<IProperties> globals = createProperties(true);
  451. for (int i = 1; i < argc; i++)
  452. globals->loadProp(argv[i], true);
  453. Owned<IFile> sentinelFile = createSentinelTarget();
  454. removeSentinelFile(sentinelFile);
  455. if (globals->hasProp("--topology"))
  456. globals->getProp("--topology", topologyFile);
  457. else
  458. topologyFile.append(codeDirectory).append(PATHSEPCHAR).append("RoxieTopology.xml");
  459. if (checkFileExists(topologyFile.str()))
  460. {
  461. DBGLOG("Loading topology file %s", topologyFile.str());
  462. topology = createPTreeFromXMLFile(topologyFile.str(), ipt_lowmem);
  463. saveTopology();
  464. }
  465. else
  466. {
  467. if (globals->hasProp("--topology"))
  468. {
  469. // Explicitly-named topology file SHOULD exist...
  470. throw MakeStringException(ROXIE_INVALID_TOPOLOGY, "topology file %s not found", topologyFile.str());
  471. }
  472. topology=createPTreeFromXMLString(
  473. "<RoxieTopology allFilesDynamic='1' localSlave='1' resolveLocally='1'>"
  474. " <RoxieFarmProcess/>"
  475. " <RoxieServerProcess netAddress='.'/>"
  476. "</RoxieTopology>"
  477. , ipt_lowmem
  478. );
  479. int port = globals->getPropInt("--port", 9876);
  480. topology->setPropInt("RoxieFarmProcess/@port", port);
  481. topology->setProp("@daliServers", globals->queryProp("--daliServers"));
  482. topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
  483. topology->setPropInt("@allFilesDynamic", globals->getPropInt("--allFilesDynamic", 1));
  484. topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
  485. topology->setProp("@disableLocalOptimizations", globals->queryProp("--disableLocalOptimizations"));
  486. }
  487. if (topology->hasProp("PreferredCluster"))
  488. {
  489. preferredClusters = new MapStringTo<int>(true);
  490. Owned<IPropertyTreeIterator> clusters = topology->getElements("PreferredCluster");
  491. ForEach(*clusters)
  492. {
  493. IPropertyTree &child = clusters->query();
  494. const char *name = child.queryProp("@name");
  495. int priority = child.getPropInt("@priority", 100);
  496. if (name && *name)
  497. preferredClusters->setValue(name, priority);
  498. }
  499. }
  500. topology->getProp("@name", roxieName);
  501. if (roxieName.length())
  502. setStatisticsComponentName(SCTroxie, roxieName, true);
  503. else
  504. setStatisticsComponentName(SCTroxie, "roxie", true);
  505. Owned<const IQueryDll> standAloneDll;
  506. if (globals->hasProp("--loadWorkunit"))
  507. {
  508. StringBuffer workunitName;
  509. globals->getProp("--loadWorkunit", workunitName);
  510. standAloneDll.setown(createQueryDll(workunitName));
  511. }
  512. else
  513. {
  514. Owned<ILoadedDllEntry> dll = createExeDllEntry(argv[0]);
  515. if (checkEmbeddedWorkUnitXML(dll))
  516. {
  517. standAloneDll.setown(createExeQueryDll(argv[0]));
  518. runOnce = globals->getPropInt("--port", 0) == 0;
  519. }
  520. }
  521. traceLevel = topology->getPropInt("@traceLevel", runOnce ? 0 : 1);
  522. if (traceLevel > MAXTRACELEVEL)
  523. traceLevel = MAXTRACELEVEL;
  524. udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1);
  525. roxiemem::setMemTraceLevel(topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1));
  526. soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
  527. miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0);
  528. Linked<IPropertyTree> directoryTree = topology->queryPropTree("Directories");
  529. if (!directoryTree)
  530. {
  531. Owned<IPropertyTree> envFile = getHPCCEnvironment();
  532. if (envFile)
  533. directoryTree.set(envFile->queryPropTree("Software/Directories"));
  534. }
  535. if (directoryTree)
  536. {
  537. getConfigurationDirectory(directoryTree, "query", "roxie", roxieName, queryDirectory);
  538. for (unsigned replicationLevel = 0; replicationLevel < MAX_REPLICATION_LEVELS; replicationLevel++)
  539. {
  540. StringBuffer dataDir;
  541. StringBuffer dirId("data");
  542. if (replicationLevel)
  543. dirId.append(replicationLevel+1);
  544. if (getConfigurationDirectory(directoryTree, dirId, "roxie", roxieName, dataDir))
  545. setBaseDirectory(dataDir, replicationLevel, DFD_OSdefault);
  546. }
  547. }
  548. directoryTree.clear();
  549. //Logging stuff
  550. if (globals->getPropBool("--stdlog", traceLevel != 0) || topology->getPropBool("@forceStdLog", false))
  551. queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_milliTime | MSGFIELD_thread | MSGFIELD_prefix);
  552. else
  553. removeLog();
  554. if (globals->hasProp("--logfile"))
  555. {
  556. Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(topology, "roxie");
  557. lf->setMaxDetail(TopDetail);
  558. lf->beginLogging();
  559. logDirectory.set(lf->queryLogDir());
  560. #ifdef _DEBUG
  561. unsigned useLogQueue = topology->getPropBool("@useLogQueue", false);
  562. #else
  563. unsigned useLogQueue = topology->getPropBool("@useLogQueue", true);
  564. #endif
  565. if (useLogQueue)
  566. {
  567. unsigned logQueueLen = topology->getPropInt("@logQueueLen", 512);
  568. unsigned logQueueDrop = topology->getPropInt("@logQueueDrop", 32);
  569. queryLogMsgManager()->enterQueueingMode();
  570. queryLogMsgManager()->setQueueDroppingLimit(logQueueLen, logQueueDrop);
  571. }
  572. if (globals->getPropBool("--enableSysLog",true))
  573. UseSysLogForOperatorMessages();
  574. }
  575. roxieMetrics.setown(createRoxieMetricsManager());
  576. Owned<IPropertyTreeIterator> userMetrics = topology->getElements("./UserMetric");
  577. ForEach(*userMetrics)
  578. {
  579. IPropertyTree &metric = userMetrics->query();
  580. const char *name = metric.queryProp("@name");
  581. const char *regex= metric.queryProp("@regex");
  582. if (name && regex)
  583. roxieMetrics->addUserMetric(name, regex);
  584. else
  585. throw MakeStringException(ROXIE_INTERNAL_ERROR, "Invalid UserMetric element in topology file - name or regex missing");
  586. }
  587. restarts = globals->getPropInt("--restarts", 0);
  588. const char *preferredSubnet = topology->queryProp("@preferredSubnet");
  589. if (preferredSubnet)
  590. {
  591. const char *preferredSubnetMask = topology->queryProp("@preferredSubnetMask");
  592. if (!preferredSubnetMask) preferredSubnetMask = "255.255.255.0";
  593. if (!setPreferredSubnet(preferredSubnet, preferredSubnetMask))
  594. throw MakeStringException(ROXIE_INTERNAL_ERROR, "Error setting preferred subnet %s mask %s", preferredSubnet, preferredSubnetMask);
  595. }
  596. if (restarts)
  597. {
  598. if (traceLevel)
  599. DBGLOG("Roxie restarting: restarts = %d build = %s", restarts, BUILD_TAG);
  600. setStartRuid(restarts);
  601. }
  602. else
  603. {
  604. if (traceLevel)
  605. {
  606. DBGLOG("Roxie starting, build = %s", BUILD_TAG);
  607. }
  608. }
  609. headRegionSize = topology->getPropInt("@headRegionSize", 50);
  610. ccdMulticastPort = topology->getPropInt("@multicastPort", CCD_MULTICAST_PORT);
  611. statsExpiryTime = topology->getPropInt("@statsExpiryTime", 3600);
  612. roxiemem::setMemTraceSizeLimit((memsize_t) topology->getPropInt64("@memTraceSizeLimit", 0));
  613. callbackRetries = topology->getPropInt("@callbackRetries", 3);
  614. callbackTimeout = topology->getPropInt("@callbackTimeout", 5000);
  615. lowTimeout = topology->getPropInt("@lowTimeout", 10000);
  616. highTimeout = topology->getPropInt("@highTimeout", 2000);
  617. slaTimeout = topology->getPropInt("@slaTimeout", 2000);
  618. parallelLoopFlowLimit = topology->getPropInt("@parallelLoopFlowLimit", 100);
  619. perChannelFlowLimit = topology->getPropInt("@perChannelFlowLimit", 10);
  620. copyResources = topology->getPropBool("@copyResources", true);
  621. useRemoteResources = topology->getPropBool("@useRemoteResources", true);
  622. checkFileDate = topology->getPropBool("@checkFileDate", true);
  623. const char *lazyOpenMode = topology->queryProp("@lazyOpen");
  624. if (!lazyOpenMode || stricmp(lazyOpenMode, "smart")==0)
  625. lazyOpen = (restarts > 0);
  626. else
  627. lazyOpen = topology->getPropBool("@lazyOpen", false);
  628. bool useNasTranslation = topology->getPropBool("@useNASTranslation", true);
  629. if (useNasTranslation)
  630. {
  631. Owned<IPropertyTree> nas = envGetNASConfiguration(topology);
  632. envInstallNASHooks(nas);
  633. }
  634. localSlave = topology->getPropBool("@localSlave", false);
  635. doIbytiDelay = topology->getPropBool("@doIbytiDelay", true);
  636. minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
  637. initIbytiDelay = topology->getPropInt("@initIbytiDelay", 50);
  638. allFilesDynamic = topology->getPropBool("@allFilesDynamic", false);
  639. lockSuperFiles = topology->getPropBool("@lockSuperFiles", false);
  640. ignoreOrphans = topology->getPropBool("@ignoreOrphans", true);
  641. chunkingHeap = topology->getPropBool("@chunkingHeap", true);
  642. readTimeout = topology->getPropInt("@readTimeout", 300);
  643. logFullQueries = topology->getPropBool("@logFullQueries", false);
  644. debugPermitted = topology->getPropBool("@debugPermitted", true);
  645. blindLogging = topology->getPropBool("@blindLogging", false);
  646. if (!blindLogging)
  647. logExcessiveSeeks = true;
  648. preloadOnceData = topology->getPropBool("@preloadOnceData", true);
  649. reloadRetriesFailed = topology->getPropBool("@reloadRetriesSuspended", true);
  650. #if defined(__linux__) && defined(SYS_ioprio_set)
  651. const char *backgroundCopyClassString = topology->queryProp("@backgroundCopyClass");
  652. if (!isEmptyString(backgroundCopyClassString))
  653. {
  654. if (strieq(backgroundCopyClassString, "best-effort"))
  655. backgroundCopyClass = IOPRIO_CLASS_BE;
  656. else if (strieq(backgroundCopyClassString, "idle"))
  657. backgroundCopyClass = IOPRIO_CLASS_IDLE;
  658. else if (strieq(backgroundCopyClassString, "none"))
  659. backgroundCopyClass = IOPRIO_CLASS_NONE;
  660. else
  661. DBGLOG("Invalid backgroundCopyClass %s specified - ignored", backgroundCopyClassString);
  662. }
  663. backgroundCopyPrio = topology->getPropInt("@backgroundCopyPrio", 0);
  664. if (backgroundCopyPrio >= IOPRIO_BE_NR)
  665. {
  666. DBGLOG("Invalid backgroundCopyPrio %d specified - using %d", backgroundCopyPrio, (int) (IOPRIO_BE_NR-1));
  667. backgroundCopyPrio = IOPRIO_BE_NR-1;
  668. }
  669. else if (backgroundCopyPrio < 0)
  670. {
  671. DBGLOG("Invalid backgroundCopyPrio %d specified - using 0", backgroundCopyPrio);
  672. backgroundCopyPrio = 0;
  673. }
  674. #endif
  675. linuxYield = topology->getPropBool("@linuxYield", false);
  676. traceSmartStepping = topology->getPropBool("@traceSmartStepping", false);
  677. useMemoryMappedIndexes = topology->getPropBool("@useMemoryMappedIndexes", false);
  678. flushJHtreeCacheOnOOM = topology->getPropBool("@flushJHtreeCacheOnOOM", true);
  679. fastLaneQueue = topology->getPropBool("@fastLaneQueue", true);
  680. udpOutQsPriority = topology->getPropInt("@udpOutQsPriority", 0);
  681. udpSnifferEnabled = topology->getPropBool("@udpSnifferEnabled", true);
  682. udpInlineCollation = topology->getPropBool("@udpInlineCollation", false);
  683. udpInlineCollationPacketLimit = topology->getPropInt("@udpInlineCollationPacketLimit", 50);
  684. udpRetryBusySenders = topology->getPropInt("@udpRetryBusySenders", 0);
  685. // Historically, this was specified in seconds. Assume any value <= 10 is a legacy value specified in seconds!
  686. udpMaxRetryTimedoutReqs = topology->getPropInt("@udpMaxRetryTimedoutReqs", 0);
  687. udpRequestToSendTimeout = topology->getPropInt("@udpRequestToSendTimeout", 0);
  688. if (udpRequestToSendTimeout<=10)
  689. udpRequestToSendTimeout *= 1000;
  690. if (udpRequestToSendTimeout == 0)
  691. {
  692. if (slaTimeout)
  693. udpRequestToSendTimeout = (slaTimeout*3) / 4;
  694. else
  695. udpRequestToSendTimeout = 5000;
  696. }
  697. // MORE: might want to check socket buffer sizes against sys max here instead of udp threads ?
  698. udpSnifferReadThreadPriority = topology->getPropInt("@udpSnifferReadThreadPriority", 3);
  699. udpSnifferSendThreadPriority = topology->getPropInt("@udpSnifferSendThreadPriority", 3);
  700. udpMulticastBufferSize = topology->getPropInt("@udpMulticastBufferSize", 262142);
  701. udpFlowSocketsSize = topology->getPropInt("@udpFlowSocketsSize", 131072);
  702. udpLocalWriteSocketSize = topology->getPropInt("@udpLocalWriteSocketSize", 1024000);
  703. roxieMulticastEnabled = topology->getPropBool("@roxieMulticastEnabled", true); // enable use of multicast for sending requests to slaves
  704. if (udpSnifferEnabled && !roxieMulticastEnabled)
  705. {
  706. DBGLOG("WARNING: ignoring udpSnifferEnabled setting as multicast not enabled");
  707. udpSnifferEnabled = false;
  708. }
  709. int ttlTmp = topology->getPropInt("@multicastTTL", 1);
  710. if (ttlTmp < 0)
  711. {
  712. multicastTTL = 1;
  713. WARNLOG("multicastTTL value (%d) invalid, must be >=0, resetting to %u", ttlTmp, multicastTTL);
  714. }
  715. else if (ttlTmp > 255)
  716. {
  717. multicastTTL = 255;
  718. WARNLOG("multicastTTL value (%d) invalid, must be <=%u, resetting to maximum", ttlTmp, multicastTTL);
  719. }
  720. else
  721. multicastTTL = ttlTmp;
  722. indexReadChunkSize = topology->getPropInt("@indexReadChunkSize", 60000);
  723. numSlaveThreads = topology->getPropInt("@slaveThreads", 30);
  724. numServerThreads = topology->getPropInt("@serverThreads", 30);
  725. numRequestArrayThreads = topology->getPropInt("@requestArrayThreads", 5);
  726. maxBlockSize = topology->getPropInt("@maxBlockSize", 10000000);
  727. maxLockAttempts = topology->getPropInt("@maxLockAttempts", 5);
  728. enableHeartBeat = topology->getPropBool("@enableHeartBeat", true);
  729. checkCompleted = topology->getPropBool("@checkCompleted", true);
  730. prestartSlaveThreads = topology->getPropBool("@prestartSlaveThreads", false);
  731. preabortKeyedJoinsThreshold = topology->getPropInt("@preabortKeyedJoinsThreshold", 100);
  732. preabortIndexReadsThreshold = topology->getPropInt("@preabortIndexReadsThreshold", 100);
  733. defaultMemoryLimit = (memsize_t) topology->getPropInt64("@defaultMemoryLimit", 0);
  734. defaultTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeLimit", 0);
  735. defaultTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeLimit", 0);
  736. defaultTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeLimit", 0);
  737. defaultWarnTimeLimit[0] = (unsigned) topology->getPropInt64("@defaultLowPriorityTimeWarning", 0);
  738. defaultWarnTimeLimit[1] = (unsigned) topology->getPropInt64("@defaultHighPriorityTimeWarning", 0);
  739. defaultWarnTimeLimit[2] = (unsigned) topology->getPropInt64("@defaultSLAPriorityTimeWarning", 0);
  740. defaultThorConnectTimeout = (unsigned) topology->getPropInt64("@defaultThorConnectTimeout", 60);
  741. defaultXmlReadFlags = topology->getPropBool("@defaultStripLeadingWhitespace", true) ? ptr_ignoreWhiteSpace : ptr_none;
  742. defaultParallelJoinPreload = topology->getPropInt("@defaultParallelJoinPreload", 0);
  743. defaultConcatPreload = topology->getPropInt("@defaultConcatPreload", 0);
  744. defaultFetchPreload = topology->getPropInt("@defaultFetchPreload", 0);
  745. defaultFullKeyedJoinPreload = topology->getPropInt("@defaultFullKeyedJoinPreload", 0);
  746. defaultKeyedJoinPreload = topology->getPropInt("@defaultKeyedJoinPreload", 0);
  747. defaultPrefetchProjectPreload = topology->getPropInt("@defaultPrefetchProjectPreload", 10);
  748. defaultStrandBlockSize = topology->getPropInt("@defaultStrandBlockSize", 512);
  749. defaultForceNumStrands = topology->getPropInt("@defaultForceNumStrands", 0);
  750. defaultCheckingHeap = topology->getPropBool("@checkingHeap", false); // NOTE - not in configmgr - too dangerous!
  751. defaultDisableLocalOptimizations = topology->getPropBool("@disableLocalOptimizations", false); // NOTE - not in configmgr - too dangerous!
  752. slaveQueryReleaseDelaySeconds = topology->getPropInt("@slaveQueryReleaseDelaySeconds", 60);
  753. coresPerQuery = topology->getPropInt("@coresPerQuery", 0);
  754. diskReadBufferSize = topology->getPropInt("@diskReadBufferSize", 0x10000);
  755. fieldTranslationEnabled = RecordTranslationMode::None;
  756. const char *val = topology->queryProp("@fieldTranslationEnabled");
  757. if (val)
  758. fieldTranslationEnabled = getTranslationMode(val);
  759. pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
  760. memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);
  761. roxiemem::setMemoryStatsInterval(memoryStatsInterval);
  762. pingInterval = topology->getPropInt("@pingInterval", 0);
  763. socketCheckInterval = topology->getPropInt("@socketCheckInterval", runOnce ? 0 : 5000);
  764. memsize_t totalMemoryLimit = (memsize_t) topology->getPropInt64("@totalMemoryLimit", 0);
  765. bool allowHugePages = topology->getPropBool("@heapUseHugePages", false);
  766. bool allowTransparentHugePages = topology->getPropBool("@heapUseTransparentHugePages", true);
  767. bool retainMemory = topology->getPropBool("@heapRetainMemory", false);
  768. if (!totalMemoryLimit)
  769. totalMemoryLimit = 1024 * 0x100000; // 1 Gb;
  770. roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, retainMemory, totalMemoryLimit, 0, NULL, NULL);
  771. traceStartStop = topology->getPropBool("@traceStartStop", false);
  772. traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);
  773. defaultTimeActivities = topology->getPropBool("@timeActivities", true);
  774. defaultTraceEnabled = topology->getPropBool("@traceEnabled", false);
  775. defaultTraceLimit = topology->getPropInt("@traceLimit", 10);
  776. clientCert.certificate.set(topology->queryProp("@certificateFileName"));
  777. clientCert.privateKey.set(topology->queryProp("@privateKeyFileName"));
  778. clientCert.passphrase.set(topology->queryProp("@passphrase"));
  779. useHardLink = topology->getPropBool("@useHardLink", false);
  780. maxFileAge[false] = topology->getPropInt("@localFilesExpire", (unsigned) -1);
  781. maxFileAge[true] = topology->getPropInt("@remoteFilesExpire", 60*60*1000);
  782. minFilesOpen[false] = topology->getPropInt("@minLocalFilesOpen", 2000);
  783. minFilesOpen[true] = topology->getPropInt("@minRemoteFilesOpen", 500);
  784. maxFilesOpen[false] = topology->getPropInt("@maxLocalFilesOpen", 4000);
  785. maxFilesOpen[true] = topology->getPropInt("@maxRemoteFilesOpen", 1000);
  786. dafilesrvLookupTimeout = topology->getPropInt("@dafilesrvLookupTimeout", 10000);
  787. setRemoteFileTimeouts(dafilesrvLookupTimeout, 0);
  788. topology->getProp("@daliServers", fileNameServiceDali);
  789. trapTooManyActiveQueries = topology->getPropBool("@trapTooManyActiveQueries", true);
  790. maxEmptyLoopIterations = topology->getPropInt("@maxEmptyLoopIterations", 1000);
  791. maxGraphLoopIterations = topology->getPropInt("@maxGraphLoopIterations", 1000);
  792. mergeSlaveStatistics = topology->getPropBool("@mergeSlaveStatistics", true);
  793. enableKeyDiff = topology->getPropBool("@enableKeyDiff", true);
  794. // MORE: Get parms from topology after it is populated from Hardware/computer types section in configenv
  795. // Then if does not match and based on desired action in topolgy, either warn, or fatal exit or .... etc
  796. // Also get prim path and sec from topology
  797. #ifdef _WIN32
  798. getHardwareInfo(hdwInfo, "C:", "D:");
  799. #else // linux
  800. getHardwareInfo(hdwInfo, "/c$", "/d$");
  801. #endif
  802. if (traceLevel)
  803. {
  804. DBGLOG("Current Hardware Info: CPUs=%i, speed=%i MHz, Mem=%i MB , primDisk=%i GB, primFree=%i GB, secDisk=%i GB, secFree=%i GB, NIC=%i",
  805. hdwInfo.numCPUs, hdwInfo.CPUSpeed, hdwInfo.totalMemory,
  806. hdwInfo.primDiskSize, hdwInfo.primFreeSize, hdwInfo.secDiskSize, hdwInfo.secFreeSize, hdwInfo.NICSpeed);
  807. }
  808. parallelAggregate = topology->getPropInt("@parallelAggregate", 0);
  809. if (!parallelAggregate)
  810. parallelAggregate = hdwInfo.numCPUs;
  811. if (!parallelAggregate)
  812. parallelAggregate = 1;
  813. simpleLocalKeyedJoins = topology->getPropBool("@simpleLocalKeyedJoins", true);
  814. inMemoryKeysEnabled = topology->getPropBool("@inMemoryKeysEnabled", true);
  815. serverSideCacheSize = topology->getPropInt("@serverSideCacheSize", 0);
  816. setKeyIndexCacheSize((unsigned)-1); // unbound
  817. nodeCachePreload = topology->getPropBool("@nodeCachePreload", false);
  818. setNodeCachePreload(nodeCachePreload);
  819. nodeCacheMB = topology->getPropInt("@nodeCacheMem", 100);
  820. setNodeCacheMem(nodeCacheMB * 0x100000);
  821. leafCacheMB = topology->getPropInt("@leafCacheMem", 50);
  822. setLeafCacheMem(leafCacheMB * 0x100000);
  823. blobCacheMB = topology->getPropInt("@blobCacheMem", 0);
  824. setBlobCacheMem(blobCacheMB * 0x100000);
  825. unsigned __int64 affinity = topology->getPropInt64("@affinity", 0);
  826. updateAffinity(affinity);
  827. minFreeDiskSpace = topology->getPropInt64("@minFreeDiskSpace", (1024 * 0x100000)); // default to 1 GB
  828. if (topology->getPropBool("@jumboFrames", false))
  829. {
  830. mtu_size = 9000; // upper limit on outbound buffer size - allow some header room too
  831. roxiemem::setDataAlignmentSize(0x2000);
  832. }
  833. else
  834. {
  835. mtu_size = 1400; // upper limit on outbound buffer size - allow some header room too
  836. roxiemem::setDataAlignmentSize(0x400);
  837. }
  838. unsigned pinterval = topology->getPropInt("@systemMonitorInterval",1000*60);
  839. perfMonHook.setown(roxiemem::createRoxieMemStatsPerfMonHook()); // Note - we create even if pinterval is 0, as can be enabled via control message
  840. if (pinterval)
  841. startPerformanceMonitor(pinterval, PerfMonStandard, perfMonHook);
  842. topology->getProp("@pluginDirectory", pluginDirectory);
  843. StringBuffer packageDirectory;
  844. getPackageFolder(packageDirectory);
  845. if (pluginDirectory.length() == 0 && packageDirectory.length() != 0)
  846. {
  847. pluginDirectory.append(packageDirectory).append("plugins");
  848. }
  849. getAdditionalPluginsPath(pluginDirectory, packageDirectory);
  850. if (queryDirectory.length() == 0)
  851. {
  852. topology->getProp("@queryDir", queryDirectory);
  853. if (queryDirectory.length() == 0)
  854. queryDirectory.append(codeDirectory).append("queries");
  855. }
  856. addNonEmptyPathSepChar(queryDirectory);
  857. queryFileCache().start();
  858. getTempFilePath(tempDirectory, "roxie", topology);
  859. #ifdef _WIN32
  860. topology->addPropBool("@linuxOS", false);
  861. #else
  862. topology->addPropBool("@linuxOS", true);
  863. #endif
  864. allQuerySetNames.appendListUniq(topology->queryProp("@querySets"), ",");
  865. Owned<IPropertyTreeIterator> roxieServers = topology->getElements("./RoxieServerProcess");
  866. ForEach(*roxieServers)
  867. {
  868. IPropertyTree &roxieServer = roxieServers->query();
  869. const char *iptext = roxieServer.queryProp("@netAddress");
  870. unsigned nodeIndex = addRoxieNode(iptext);
  871. if (getNodeAddress(nodeIndex).isLocal())
  872. myNodeIndex = nodeIndex;
  873. if (traceLevel > 3)
  874. DBGLOG("Roxie server %u is at %s", nodeIndex, iptext);
  875. }
  876. if (myNodeIndex == -1)
  877. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - current node is not in server list");
  878. // Set multicast base addresses - must be done before generating slave channels
  879. if (roxieMulticastEnabled && !localSlave)
  880. {
  881. if (topology->queryProp("@multicastBase"))
  882. multicastBase.ipset(topology->queryProp("@multicastBase"));
  883. else
  884. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastBase not set");
  885. if (topology->queryProp("@multicastLast"))
  886. multicastLast.ipset(topology->queryProp("@multicastLast"));
  887. else
  888. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - multicastLast not set");
  889. }
  890. // Generate the slave channels
  891. unsigned numDataCopies = topology->getPropInt("@numDataCopies", 1);
  892. if (!numDataCopies)
  893. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numDataCopies should be > 0");
  894. unsigned channelsPerNode = topology->getPropInt("@channelsPerNode", 1);
  895. unsigned numNodes = getNumNodes();
  896. const char *slaveConfig = topology->queryProp("@slaveConfig");
  897. if (!slaveConfig)
  898. slaveConfig = "simple";
  899. if (strnicmp(slaveConfig, "cyclic", 6) == 0)
  900. {
  901. numChannels = numNodes;
  902. unsigned cyclicOffset = topology->getPropInt("@cyclicOffset", 1);
  903. for (unsigned i=0; i<numNodes; i++)
  904. {
  905. // Note this code is a little confusing - easy to get the cyclic offset backwards
  906. // cyclic offset means node n+offset has copy 2 for channel n, so node n has copy 2 for channel n-offset
  907. int channel = (int)i+1;
  908. for (unsigned copy=0; copy<numDataCopies; copy++)
  909. {
  910. if (channel < 1)
  911. channel = channel + numNodes;
  912. addChannel(i, channel, copy);
  913. channel -= cyclicOffset;
  914. }
  915. }
  916. }
  917. else if (strnicmp(slaveConfig, "overloaded", 10) == 0)
  918. {
  919. if (!channelsPerNode)
  920. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - channelsPerNode should be > 0");
  921. numChannels = numNodes * channelsPerNode;
  922. for (unsigned i=0; i<numNodes; i++)
  923. {
  924. int channel = (int)(i+1);
  925. for (unsigned copy=0; copy<channelsPerNode; copy++)
  926. {
  927. addChannel(i, channel, copy);
  928. channel += numNodes;
  929. }
  930. }
  931. }
  932. else // 'Full redundancy' or 'simple' mode
  933. {
  934. if (numNodes % numDataCopies)
  935. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels not an integer");
  936. numChannels = numNodes / numDataCopies;
  937. int channel = 1;
  938. for (unsigned i=0; i<numNodes; i++)
  939. {
  940. addChannel(i, channel, 0);
  941. channel++;
  942. if ((unsigned)channel > numChannels)
  943. channel = 1;
  944. }
  945. }
  946. if (!numChannels)
  947. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - numChannels calculated at 0");
  948. if (numChannels > 1 && localSlave)
  949. throw MakeStringException(MSGAUD_operator, ROXIE_INVALID_TOPOLOGY, "Invalid topology file - localSlave requires single channel (%d channels specified)", numChannels);
  950. // Now we know all the channels, we can open and subscribe the multicast channels
  951. if (!localSlave)
  952. openMulticastSocket();
  953. setDaliServixSocketCaching(true); // enable daliservix caching
  954. loadPlugins();
  955. createDelayedReleaser();
  956. globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
  957. globalPackageSetManager->load();
  958. unsigned snifferChannel = numChannels+2; // MORE - why +2 not +1 ??
  959. ROQ = createOutputQueueManager(snifferChannel, numSlaveThreads);
  960. ROQ->setHeadRegionSize(headRegionSize);
  961. ROQ->start();
  962. Owned<IPacketDiscarder> packetDiscarder = createPacketDiscarder();
  963. #if defined(WIN32) && defined(_DEBUG) && defined(_DEBUG_HEAP_FULL)
  964. int tmpFlag = _CrtSetDbgFlag( _CRTDBG_REPORT_FLAG );
  965. tmpFlag |= _CRTDBG_CHECK_ALWAYS_DF;
  966. _CrtSetDbgFlag( tmpFlag );
  967. #endif
  968. EnableSEHtoExceptionMapping();
  969. setSEHtoExceptionHandler(&abortHandler);
  970. Owned<IHpccProtocolPluginContext> protocolCtx = new CHpccProtocolPluginCtx();
  971. if (runOnce)
  972. {
  973. if (globals->hasProp("-wu"))
  974. {
  975. Owned<IHpccProtocolListener> roxieServer = createRoxieWorkUnitListener(1, false);
  976. try
  977. {
  978. VStringBuffer x("-%s", argv[0]);
  979. roxieServer->runOnce(x);
  980. fflush(stdout); // in windows if output is redirected results don't appear without flushing
  981. }
  982. catch (IException *E)
  983. {
  984. EXCLOG(E);
  985. E->Release();
  986. }
  987. }
  988. else
  989. {
  990. Owned<IHpccProtocolPlugin> protocolPlugin = loadHpccProtocolPlugin(protocolCtx, NULL);
  991. Owned<IHpccProtocolListener> roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(getNodeAddress(myNodeIndex), 0, 1, false), 0, 0, NULL);
  992. try
  993. {
  994. const char *format = globals->queryProp("format");
  995. if (!format)
  996. {
  997. if (globals->hasProp("-xml"))
  998. format = "xml";
  999. else if (globals->hasProp("-csv"))
  1000. format = "csv";
  1001. else if (globals->hasProp("-raw"))
  1002. format = "raw";
  1003. else
  1004. format = "ascii";
  1005. }
  1006. StringBuffer query;
  1007. query.appendf("<roxie format='%s'/>", format);
  1008. roxieServer->runOnce(query.str()); // MORE - should use the wu listener instead I suspect
  1009. fflush(stdout); // in windows if output is redirected results don't appear without flushing
  1010. }
  1011. catch (IException *E)
  1012. {
  1013. EXCLOG(E);
  1014. E->Release();
  1015. }
  1016. }
  1017. }
  1018. else
  1019. {
  1020. Owned<IPropertyTreeIterator> roxieFarms = topology->getElements("./RoxieFarmProcess");
  1021. ForEach(*roxieFarms)
  1022. {
  1023. IPropertyTree &roxieFarm = roxieFarms->query();
  1024. unsigned listenQueue = roxieFarm.getPropInt("@listenQueue", DEFAULT_LISTEN_QUEUE_SIZE);
  1025. unsigned numThreads = roxieFarm.getPropInt("@numThreads", numServerThreads);
  1026. unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
  1027. unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
  1028. // NOTE: farmer name [@name=] is not copied into topology
  1029. const IpAddress &ip = getNodeAddress(myNodeIndex);
  1030. if (!roxiePort)
  1031. {
  1032. roxiePort = port;
  1033. ownEP.set(roxiePort, ip);
  1034. }
  1035. bool suspended = roxieFarm.getPropBool("@suspended", false);
  1036. Owned <IHpccProtocolListener> roxieServer;
  1037. if (port)
  1038. {
  1039. const char *protocol = roxieFarm.queryProp("@protocol");
  1040. StringBuffer certFileName;
  1041. StringBuffer keyFileName;
  1042. StringBuffer passPhraseStr;
  1043. if (protocol && streq(protocol, "ssl"))
  1044. {
  1045. #ifdef _USE_OPENSSL
  1046. const char *certFile = roxieFarm.queryProp("@certificateFileName");
  1047. if (!certFile)
  1048. throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port);
  1049. if (isAbsolutePath(certFile))
  1050. certFileName.append(certFile);
  1051. else
  1052. certFileName.append(codeDirectory.str()).append(certFile);
  1053. if (!checkFileExists(certFileName.str()))
  1054. throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str());
  1055. const char *keyFile = roxieFarm.queryProp("@privateKeyFileName");
  1056. if (!keyFile)
  1057. throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port);
  1058. if (isAbsolutePath(keyFile))
  1059. keyFileName.append(keyFile);
  1060. else
  1061. keyFileName.append(codeDirectory.str()).append(keyFile);
  1062. if (!checkFileExists(keyFileName.str()))
  1063. throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str());
  1064. const char *passPhrase = roxieFarm.queryProp("@passphrase");
  1065. if (!isEmptyString(passPhrase))
  1066. decrypt(passPhraseStr, passPhrase);
  1067. #else
  1068. WARNLOG("Skipping Roxie SSL Farm Listener on port %d : OpenSSL disabled in build", port);
  1069. continue;
  1070. #endif
  1071. }
  1072. const char *soname = roxieFarm.queryProp("@so");
  1073. const char *config = roxieFarm.queryProp("@config");
  1074. Owned<IHpccProtocolPlugin> protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname);
  1075. roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, certFileName.str(), keyFileName.str(), passPhraseStr.str()));
  1076. }
  1077. else
  1078. roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended));
  1079. IHpccProtocolMsgSink *sink = roxieServer->queryMsgSink();
  1080. const char *aclName = roxieFarm.queryProp("@aclName");
  1081. if (aclName && *aclName)
  1082. {
  1083. Owned<IPropertyTree> aclInfo = createPTree("AccessInfo", ipt_lowmem);
  1084. getAccessList(aclName, topology, aclInfo);
  1085. Owned<IPropertyTreeIterator> accesses = aclInfo->getElements("Access");
  1086. ForEach(*accesses)
  1087. {
  1088. IPropertyTree &access = accesses->query();
  1089. try
  1090. {
  1091. sink->addAccess(access.getPropBool("@allow", true), access.getPropBool("@allowBlind", true), access.queryProp("@ip"), access.queryProp("@mask"), access.queryProp("@query"), access.queryProp("@error"), access.getPropInt("@errorCode"));
  1092. }
  1093. catch (IException *E)
  1094. {
  1095. StringBuffer s, x;
  1096. E->errorMessage(s);
  1097. E->Release();
  1098. toXML(&access, x, 0, 0);
  1099. throw MakeStringException(ROXIE_ACL_ERROR, "Error in access statement %s: %s", x.str(), s.str());
  1100. }
  1101. }
  1102. }
  1103. socketListeners.append(*roxieServer.getLink());
  1104. time(&startupTime);
  1105. roxieServer->start();
  1106. }
  1107. writeSentinelFile(sentinelFile);
  1108. DBGLOG("Startup completed - LPT=%u APT=%u", queryNumLocalTrees(), queryNumAtomTrees());
  1109. DBGLOG("Waiting for queries");
  1110. if (pingInterval)
  1111. startPingTimer();
  1112. LocalIAbortHandler abortHandler(waiter);
  1113. waiter.wait();
  1114. }
  1115. shuttingDown = true;
  1116. if (pingInterval)
  1117. stopPingTimer();
  1118. setSEHtoExceptionHandler(NULL);
  1119. while (socketListeners.isItem(0))
  1120. {
  1121. socketListeners.item(0).stop(1000);
  1122. socketListeners.remove(0);
  1123. }
  1124. packetDiscarder->stop();
  1125. packetDiscarder.clear();
  1126. ROQ->stop();
  1127. ROQ->join();
  1128. ROQ->Release();
  1129. ROQ = NULL;
  1130. }
  1131. catch (IException *E)
  1132. {
  1133. StringBuffer x;
  1134. DBGLOG("EXCEPTION: (%d): %s", E->errorCode(), E->errorMessage(x).str());
  1135. E->Release();
  1136. }
  1137. roxieMetrics.clear();
  1138. stopPerformanceMonitor();
  1139. ::Release(globalPackageSetManager);
  1140. globalPackageSetManager = NULL;
  1141. stopDelayedReleaser();
  1142. cleanupPlugins();
  1143. closeMulticastSockets();
  1144. releaseSlaveDynamicFileCache();
  1145. releaseRoxieStateCache();
  1146. setDaliServixSocketCaching(false); // make sure it cleans up or you get bogus memleak reports
  1147. setNodeCaching(false); // ditto
  1148. perfMonHook.clear();
  1149. strdup("Make sure leak checking is working");
  1150. UseSysLogForOperatorMessages(false);
  1151. ExitModuleObjects();
  1152. releaseAtoms();
  1153. strdup("Make sure leak checking is working");
  1154. #ifdef _WIN32
  1155. #ifdef _DEBUG
  1156. #if 1
  1157. StringBuffer leakFileDir(logDirectory.str());
  1158. leakFileDir.append("roxieleaks.log");
  1159. HANDLE h = CreateFile(leakFileDir.str(), GENERIC_READ|GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, 0, 0);
  1160. _CrtSetReportMode( _CRT_WARN, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1161. _CrtSetReportFile( _CRT_WARN, h);
  1162. _CrtSetReportMode( _CRT_ERROR, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1163. _CrtSetReportFile( _CRT_ERROR, h);
  1164. _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE|_CRTDBG_MODE_DEBUG);
  1165. _CrtSetReportFile( _CRT_ASSERT, h);
  1166. // _CrtDumpMemoryLeaks(); if you uncomment these lines you get to see the leaks sooner (so can look in debugger at full memory)
  1167. // CloseHandle(h); but there will be additional leaks reported that are not really leaks
  1168. #endif
  1169. #endif
  1170. #endif
  1171. return 0;
  1172. }