loggingagent.cpp 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "httpclient.hpp"
  14. #include "ws_loggingservice_esp.ipp"
  15. #include "LoggingErrors.hpp"
  16. #include "loggingcommon.hpp"
  17. #include "loggingagentbase.hpp"
  18. #include "loggingagent.hpp"
  19. static const int DefaultMaxTriesGTS = -1;
  20. static const char* const PropESPServer = "ESPServer";
  21. static const char* const PropServerUrl = "@url";
  22. static const char* const PropServerUserID = "@user";
  23. static const char* const PropServerPassword = "@password";
  24. static const char* const PropServerWaitingSeconds = "MaxServerWaitingSeconds";
  25. static const char* const PropMaxTransIDLength = "MaxTransIDLength";
  26. static const char* const PropMaxTransIDSequenceNumber = "MaxTransIDSequenceNumber";
  27. static const char* const PropMaxTransSeedTimeoutMinutes = "MaxTransSeedTimeoutMinutes";
  28. static const char* const PropTransactionSeedType = "TransactionSeedType";
  29. static const char* const PropAlternativeTransactionSeedType = "AlternativeTransactionSeedType";
  30. static const char* const DefaultTransactionSeedType = "-";
  31. static const char* const DefaultAlternativeTransactionSeedType = "-X"; //local
  32. static const char* const MaxTriesGTS = "MaxTriesGTS";
  33. static const char* const appESPServerLoggingAgent = "ESPServerLoggingAgent";
  34. bool CESPServerLoggingAgent::init(const char * name, const char * type, IPropertyTree * cfg, const char * process)
  35. {
  36. if (!cfg)
  37. return false;
  38. agentName.set(name);
  39. const char* servicesConfig = cfg->queryProp("@services");
  40. if (isEmptyString(servicesConfig))
  41. throw MakeStringException(-1,"No Logging Service defined for %s", agentName.get());
  42. setServices(servicesConfig);
  43. IPropertyTree* espServer = cfg->queryBranch(PropESPServer);
  44. if(!espServer)
  45. throw MakeStringException(-1,"Unable to find ESPServer settings for log agent %s:%s", name, type);
  46. const char* url = espServer->queryProp(PropServerUrl);
  47. if (url && *url)
  48. serverUrl.set(url);
  49. const char* userID = espServer->queryProp(PropServerUserID);
  50. const char* password = espServer->queryProp(PropServerPassword);
  51. if (userID && *userID && password && *password)
  52. {
  53. serverUserID.set(userID);
  54. decrypt(serverPassword, password);
  55. }
  56. maxServerWaitingSeconds = cfg->getPropInt(PropServerWaitingSeconds);
  57. if (hasService(LGSTUpdateLOG))
  58. {
  59. BoolHash uniqueGroupNames;
  60. StringBuffer sourceName, groupName, dbName;
  61. Owned<IPropertyTreeIterator> iter = cfg->getElements("LogSourceMap/LogSource");
  62. ForEach(*iter)
  63. {
  64. ensureInputString(iter->query().queryProp("@name"), false, sourceName, -1, "LogSource @name required");
  65. ensureInputString(iter->query().queryProp("@maptologgroup"), true, groupName, -1, "LogSource @maptologgroup required");
  66. ensureInputString(iter->query().queryProp("@maptodb"), true, dbName, -1, "LogSource @maptodb required");
  67. Owned<CLogSource> logSource = new CLogSource(sourceName.str(), groupName.str(), dbName.str());
  68. logSources.setValue(sourceName.str(), logSource);
  69. bool* found = uniqueGroupNames.getValue(groupName.str());
  70. if (!found || !*found)
  71. {
  72. uniqueGroupNames.setValue(groupName.str(), true);
  73. unsigned maxSeq = 0;
  74. unsigned maxLength = 0;
  75. unsigned seedExpiredSeconds = 0;
  76. VStringBuffer xpath("LogGroup/[@name='%s']", groupName.str());
  77. IPropertyTree* logGroup = cfg->queryBranch(xpath.str());
  78. if (logGroup)
  79. {
  80. maxLength = logGroup->getPropInt(PropMaxTransIDLength, 0);
  81. maxSeq = logGroup->getPropInt(PropMaxTransIDSequenceNumber, 0),
  82. seedExpiredSeconds = 60 * logGroup->getPropInt(PropMaxTransSeedTimeoutMinutes, 0);
  83. }
  84. if (!hasService(LGSTGetTransactionSeed) && !hasService(LGSTGetTransactionID))
  85. continue;
  86. StringBuffer transactionSeed, statusMessage;
  87. getTransactionSeed(groupName.str(), transactionSeed, statusMessage);
  88. if (transactionSeed.length() > 0)
  89. {
  90. Owned<CTransIDBuilder> entry = new CTransIDBuilder(transactionSeed.str(), false, transactionSeedType.get(),
  91. maxLength, maxSeq, seedExpiredSeconds);
  92. transIDMap.setValue(groupName.str(), entry);
  93. if (iter->query().getPropBool("@default", false))
  94. defaultGroup.set(groupName.str());
  95. }
  96. else
  97. PROGLOG("Failed to get TransactionSeed for <%s>", groupName.str());
  98. }
  99. }
  100. logContentFilter.readAllLogFilters(cfg);
  101. }
  102. if (!hasService(LGSTGetTransactionSeed) && !hasService(LGSTGetTransactionID))
  103. return true;
  104. maxGTSRetries = cfg->getPropInt(MaxTriesGTS, DefaultMaxTriesGTS);
  105. transactionSeedType.set(cfg->hasProp(PropTransactionSeedType) ? cfg->queryProp(PropTransactionSeedType) :
  106. DefaultTransactionSeedType);
  107. alternativeTransactionSeedType.set(cfg->hasProp(PropAlternativeTransactionSeedType) ?
  108. cfg->queryProp(PropAlternativeTransactionSeedType) : DefaultAlternativeTransactionSeedType);
  109. StringBuffer localTransactionSeed;
  110. createLocalTransactionSeed(localTransactionSeed);
  111. Owned<CTransIDBuilder> localTransactionEntry = new CTransIDBuilder(localTransactionSeed.str(), true, alternativeTransactionSeedType.get(),
  112. cfg->getPropInt(PropMaxTransIDLength, 0), cfg->getPropInt(PropMaxTransIDSequenceNumber, 0),
  113. 60 * cfg->getPropInt(PropMaxTransSeedTimeoutMinutes, 0));
  114. transIDMap.setValue(appESPServerLoggingAgent, localTransactionEntry);
  115. return true;
  116. }
  117. void CESPServerLoggingAgent::createLocalTransactionSeed(StringBuffer& transactionSeed)
  118. {
  119. unsigned ip = queryHostIP().iphash();
  120. unsigned mstick6char = ((unsigned)usTick() & 0xFFFFFF);
  121. unsigned processId2char = ((unsigned)GetCurrentProcessId()) & 0xF;
  122. unsigned threadId1char = ((unsigned) (memsize_t) GetCurrentThreadId()) & 0xF;
  123. transactionSeed.setf("%02X%06X%X%X", ip, mstick6char, processId2char, threadId1char);
  124. }
  125. bool CESPServerLoggingAgent::getTransactionSeed(IEspGetTransactionSeedRequest& req, IEspGetTransactionSeedResponse& resp)
  126. {
  127. if (!hasService(LGSTGetTransactionSeed))
  128. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "%s: no getTransactionSeed service configured", agentName.get());
  129. StringBuffer statusMessage, transactionSeed;
  130. int statusCode = getTransactionSeed(req.getApplication(), transactionSeed, statusMessage);
  131. resp.setStatusCode(statusCode);
  132. resp.setSeedId(transactionSeed.str());
  133. if (statusMessage.length())
  134. resp.setStatusMessage(statusMessage.str());
  135. return (statusCode != -2);
  136. }
  137. int CESPServerLoggingAgent::getTransactionSeed(const char* appName, StringBuffer& transactionSeed, StringBuffer& statusMessage)
  138. {
  139. StringBuffer soapreq(
  140. "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
  141. "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
  142. " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
  143. " <soap:Body>");
  144. if (!appName || !*appName)
  145. soapreq.append("<GetTransactionSeedRequest/>");
  146. else
  147. soapreq.append("<GetTransactionSeedRequest><Application>").append(appName).append("</Application></GetTransactionSeedRequest>");
  148. soapreq.append("</soap:Body></soap:Envelope>");
  149. unsigned retry = 1;
  150. int statusCode = 0;
  151. while (1)
  152. {
  153. try
  154. {
  155. if (!getTransactionSeed(soapreq, statusCode, statusMessage, transactionSeed))
  156. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed,"Failed to get TransactionSeed");
  157. break;
  158. }
  159. catch (IException* e)
  160. {
  161. StringBuffer errorStr;
  162. statusMessage.set("Failed to get TransactionSeed: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
  163. ERRLOG("%s -- try %d", statusMessage.str(), retry);
  164. e->Release();
  165. if (retry >= maxGTSRetries)
  166. {
  167. statusCode = -2;
  168. break;
  169. }
  170. Sleep(retry*3000);
  171. retry++;
  172. }
  173. }
  174. return statusCode;
  175. }
  176. bool CESPServerLoggingAgent::getTransactionSeed(StringBuffer& soapreq, int& statusCode, StringBuffer& statusMessage, StringBuffer& seedID)
  177. {
  178. StringBuffer status, response;
  179. if (!sendHTTPRequest(soapreq, response, status) || !response.length() || !status.length())
  180. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to send Transaction Seed request to %s", serverUrl.str());
  181. if (!strieq(status, "200 OK"))
  182. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "%s", status.str());
  183. Owned<IPropertyTree> pTree = createPTreeFromXMLString(response.str());
  184. if (!pTree)
  185. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to read response from %s", serverUrl.str());
  186. //statusCode: 0 = success, -1 = failed after retries
  187. statusCode = pTree->getPropInt("soap:Body/GetTransactionSeedResponse/StatusCode");
  188. statusMessage.set(pTree->queryProp("soap:Body/GetTransactionSeedResponse/StatusMessage"));
  189. seedID.set(pTree->queryProp("soap:Body/GetTransactionSeedResponse/SeedId"));
  190. if (statusCode || !seedID.length())
  191. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to get Transaction Seed from %s", serverUrl.str());
  192. return true;
  193. }
  194. void CESPServerLoggingAgent::resetTransSeed(CTransIDBuilder *builder, const char* groupName)
  195. {
  196. StringBuffer transactionSeed, statusMessage;
  197. if (builder->isLocalSeed())
  198. createLocalTransactionSeed(transactionSeed);
  199. else
  200. {
  201. int statusCode = getTransactionSeed(groupName, transactionSeed, statusMessage);
  202. if (!transactionSeed.length() || (statusCode != 0))
  203. {
  204. StringBuffer msg("Failed to get Transaction Seed for ");
  205. msg.append(groupName).append(". statusCode: ").append(statusCode);
  206. if (!statusMessage.isEmpty())
  207. msg.append(", ").append(statusMessage.str());
  208. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "%s", msg.str());
  209. }
  210. }
  211. builder->resetTransSeed(transactionSeed.str(), builder->isLocalSeed() ? alternativeTransactionSeedType.get() : transactionSeedType.get());
  212. };
  213. void CESPServerLoggingAgent::getTransactionID(StringAttrMapping* transFields, StringBuffer& transactionID)
  214. {
  215. if (!hasService(LGSTGetTransactionID))
  216. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "%s: no getTransactionID service configured", agentName.get());
  217. const char* groupName = nullptr;
  218. CTransIDBuilder* transIDBuilder = nullptr;
  219. StringAttr* source = nullptr;
  220. if (transFields)
  221. source = transFields->getValue(sTransactionMethod);
  222. if (source)
  223. {
  224. CLogSource* logSource = logSources.getValue(source->get());
  225. if (logSource)
  226. {
  227. groupName = logSource->getGroupName();
  228. transIDBuilder = transIDMap.getValue(groupName);
  229. }
  230. }
  231. if (!transIDBuilder && (defaultGroup.length() != 0))
  232. {
  233. groupName = defaultGroup.str();
  234. transIDBuilder = transIDMap.getValue(groupName);
  235. }
  236. if (!transIDBuilder)
  237. {
  238. groupName = appESPServerLoggingAgent;
  239. transIDBuilder = transIDMap.getValue(appESPServerLoggingAgent);
  240. }
  241. if (!transIDBuilder) //This should not happen.
  242. throw MakeStringException(EspLoggingErrors::GetTransactionSeedFailed, "Failed to getTransactionID");
  243. if (!transIDBuilder->checkMaxSequenceNumber() || !transIDBuilder->checkTimeout())
  244. resetTransSeed(transIDBuilder, groupName);
  245. transIDBuilder->getTransID(transFields, transactionID);
  246. if (!transIDBuilder->checkMaxLength(transactionID.length()))
  247. {
  248. resetTransSeed(transIDBuilder, groupName);
  249. transIDBuilder->getTransID(transFields, transactionID);
  250. }
  251. return;
  252. }
  253. bool CESPServerLoggingAgent::updateLog(IEspUpdateLogRequestWrap& req, IEspUpdateLogResponse& resp)
  254. {
  255. try
  256. {
  257. if (!hasService(LGSTUpdateLOG))
  258. throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s: no updateLog service configured", agentName.get());
  259. StringBuffer soapreq(
  260. "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
  261. "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""
  262. " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding/\">"
  263. " <soap:Body>"
  264. );
  265. soapreq.append(req.getUpdateLogRequest());
  266. soapreq.append("</soap:Body></soap:Envelope>");
  267. StringBuffer status, respStr;
  268. if (sendHTTPRequest(soapreq, respStr, status) && status.length() && strieq(status, "200 OK"))
  269. resp.setStatusCode(0);
  270. else if (status.length() && !strieq(status, "200 OK"))
  271. throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s", status.str());
  272. else if (respStr.length())
  273. throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "%s", respStr.str());
  274. else
  275. throw MakeStringException(EspLoggingErrors::UpdateLogFailed, "Failed to send update log request to %s", serverUrl.str());
  276. }
  277. catch (IException* e)
  278. {//retry will be in update log queue.
  279. StringBuffer errorStr, errorMessage;
  280. errorMessage.append("Failed to update log: error code ").append(e->errorCode()).append(", error message ").append(e->errorMessage(errorStr));
  281. ERRLOG("%s", errorMessage.str());
  282. resp.setStatusCode(-1);
  283. resp.setStatusMessage(errorMessage.str());
  284. e->Release();
  285. }
  286. return true;
  287. }
  288. IEspUpdateLogRequestWrap* CESPServerLoggingAgent::filterLogContent(IEspUpdateLogRequestWrap* req)
  289. {
  290. return logContentFilter.filterLogContent(req);
  291. }
  292. bool CESPServerLoggingAgent::sendHTTPRequest(StringBuffer& req, StringBuffer &resp, StringBuffer &status)
  293. {
  294. Owned<IHttpClientContext> httpctx = getHttpClientContext();
  295. Owned <IHttpClient> httpclient = httpctx->createHttpClient(NULL, serverUrl.str());
  296. if (serverUserID.length() && serverPassword.length())
  297. {
  298. httpclient->setUserID(serverUserID.str());
  299. httpclient->setPassword(serverPassword.str());
  300. }
  301. httpclient->setTimeOut(maxServerWaitingSeconds);
  302. return !httpclient->sendRequest("POST", "text/xml", req, resp, status);
  303. }
  304. extern "C"
  305. {
  306. ESPSERVERLOGGINGAGENT_API IEspLogAgent* newLoggingAgent()
  307. {
  308. return new CESPServerLoggingAgent();
  309. }
  310. }