thkeydiff.cpp 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jiface.hpp"
  14. #include "dadfs.hpp"
  15. #include "thorfile.hpp"
  16. #include "eclhelper.hpp"
  17. #include "thexception.hpp"
  18. #include "thkeydiff.ipp"
  19. class CKeyDiffMaster : public CMasterActivity
  20. {
  21. IHThorKeyDiffArg *helper;
  22. Owned<IDistributedFile> originalIndexFile, newIndexFile;
  23. bool local, copyTlk;
  24. unsigned width;
  25. Owned<IFileDescriptor> patchDesc, originalDesc, newIndexDesc;
  26. StringArray clusters;
  27. StringAttr originalName, updatedName, outputName;
  28. public:
  29. CKeyDiffMaster(CMasterGraphElement *info) : CMasterActivity(info)
  30. {
  31. helper = (IHThorKeyDiffArg *)queryHelper();
  32. local = false;
  33. width = 0;
  34. copyTlk = globals->getPropBool("@diffCopyTlk", true); // because tlk can have meta data and diff/patch does not support
  35. }
  36. virtual void init()
  37. {
  38. CMasterActivity::init();
  39. OwnedRoxieString originalHelperName(helper->getOriginalName());
  40. OwnedRoxieString updatedHelperName(helper->getUpdatedName());
  41. OwnedRoxieString outputHelperName(helper->getOutputName());
  42. StringBuffer expandedFileName;
  43. queryThorFileManager().addScope(container.queryJob(), originalHelperName, expandedFileName, false);
  44. originalName.set(expandedFileName);
  45. queryThorFileManager().addScope(container.queryJob(), updatedHelperName, expandedFileName.clear(), false);
  46. updatedName.set(expandedFileName);
  47. queryThorFileManager().addScope(container.queryJob(), outputHelperName, expandedFileName.clear(), false);
  48. outputName.set(expandedFileName);
  49. originalIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), originalHelperName));
  50. newIndexFile.setown(queryThorFileManager().lookup(container.queryJob(), updatedHelperName));
  51. if (originalIndexFile->numParts() != newIndexFile->numParts())
  52. throw MakeActivityException(this, TE_KeyDiffIndexSizeMismatch, "Index %s and %s differ in width", originalName.get(), updatedName.get());
  53. if (originalIndexFile->querySuperFile() || newIndexFile->querySuperFile())
  54. throw MakeActivityException(this, 0, "Diffing super files not supported");
  55. width = originalIndexFile->numParts();
  56. originalDesc.setown(originalIndexFile->getFileDescriptor());
  57. newIndexDesc.setown(newIndexFile->getFileDescriptor());
  58. Owned<IPartDescriptor> tlkDesc = originalDesc->getPart(originalDesc->numParts()-1);
  59. const char *kind = tlkDesc->queryProperties().queryProp("@kind");
  60. local = NULL == kind || 0 != stricmp("topLevelKey", kind);
  61. if (!local)
  62. width--; // 1 part == No n distributed / Monolithic key
  63. if (width > container.queryJob().querySlaves())
  64. throw MakeActivityException(this, 0, "Unsupported: keydiff(%s, %s) - Cannot diff a key that's wider(%d) than the target cluster size(%d)", originalIndexFile->queryLogicalName(), newIndexFile->queryLogicalName(), width, container.queryJob().querySlaves());
  65. IArrayOf<IGroup> groups;
  66. fillClusterArray(container.queryJob(), outputName, clusters, groups);
  67. patchDesc.setown(queryThorFileManager().create(container.queryJob(), outputName, clusters, groups, 0 != (KDPoverwrite & helper->getFlags()), 0, !local, width));
  68. patchDesc->queryProperties().setProp("@kind", "keydiff");
  69. addReadFile(originalIndexFile);
  70. addReadFile(newIndexFile);
  71. }
  72. virtual void serializeSlaveData(MemoryBuffer &dst, unsigned slave)
  73. {
  74. if (slave < width) // if false - due to mismatch width fitting - fill in with a blank entry
  75. {
  76. dst.append(true);
  77. Owned<IPartDescriptor> originalPartDesc = originalDesc->getPart(slave);
  78. originalPartDesc->serialize(dst);
  79. Owned<IPartDescriptor> newIndexPartDesc = newIndexDesc->getPart(slave);
  80. newIndexPartDesc->serialize(dst);
  81. patchDesc->queryPart(slave)->serialize(dst);
  82. if (0 == slave)
  83. {
  84. if (!local)
  85. {
  86. dst.append(true);
  87. Owned<IPartDescriptor> originalTlkPartDesc = originalDesc->getPart(originalDesc->numParts()-1);
  88. originalTlkPartDesc->serialize(dst);
  89. Owned<IPartDescriptor> newIndexTlkPartDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
  90. newIndexTlkPartDesc->serialize(dst);
  91. patchDesc->queryPart(patchDesc->numParts()-1)->serialize(dst);
  92. }
  93. else
  94. dst.append(false);
  95. }
  96. }
  97. else
  98. dst.append(false); // no part
  99. }
  100. virtual void slaveDone(size32_t slaveIdx, MemoryBuffer &mb)
  101. {
  102. if (mb.length()) // if 0 implies aborted out from this slave.
  103. {
  104. offset_t size;
  105. mb.read(size);
  106. CDateTime modifiedTime(mb);
  107. IPartDescriptor *partDesc = patchDesc->queryPart(slaveIdx);
  108. IPropertyTree &props = partDesc->queryProperties();
  109. StringBuffer timeStr;
  110. modifiedTime.getString(timeStr);
  111. props.setProp("@modified", timeStr.str());
  112. unsigned crc;
  113. mb.read(crc);
  114. props.setPropInt64("@fileCrc", crc);
  115. if (!local && 0 == slaveIdx)
  116. {
  117. IPartDescriptor *partDesc = patchDesc->queryPart(patchDesc->numParts()-1);
  118. IPropertyTree &props = partDesc->queryProperties();
  119. mb.read(size);
  120. props.setPropInt64("@size", size);
  121. CDateTime modifiedTime(mb);
  122. StringBuffer timeStr;
  123. modifiedTime.getString(timeStr);
  124. props.setProp("@modified", timeStr.str());
  125. if (copyTlk)
  126. {
  127. Owned<IPartDescriptor> tlkDesc = newIndexDesc->getPart(newIndexDesc->numParts()-1);
  128. if (partDesc->getCrc(crc))
  129. props.setPropInt64("@fileCrc", crc);
  130. props.setProp("@diffFormat", "copy");
  131. }
  132. else
  133. {
  134. mb.read(crc);
  135. props.setPropInt64("@fileCrc", crc);
  136. props.setProp("@diffFormat", "diffV1");
  137. }
  138. }
  139. }
  140. }
  141. virtual void done()
  142. {
  143. Owned<IWorkUnit> wu = &container.queryJob().queryWorkUnit().lock();
  144. Owned<IWUResult> r = wu->updateResultBySequence(helper->getSequence());
  145. //Do not mark the result as calculated - because the patch file isn't a valid result
  146. r->setResultLogicalName(outputName);
  147. r.clear();
  148. wu.clear();
  149. IPropertyTree &patchProps = patchDesc->queryProperties();
  150. if (0 != (helper->getFlags() & KDPexpires))
  151. setExpiryTime(patchProps, helper->getExpiryDays());
  152. IPropertyTree &originalProps = originalDesc->queryProperties();;
  153. if (originalProps.queryProp("ECL"))
  154. patchProps.setProp("ECL", originalProps.queryProp("ECL"));
  155. if (originalProps.getPropBool("@local"))
  156. patchProps.setPropBool("@local", true);
  157. container.queryTempHandler()->registerFile(outputName, container.queryOwner().queryGraphId(), 0, false, WUFileStandard, &clusters);
  158. Owned<IDistributedFile> patchFile;
  159. // set part sizes etc
  160. queryThorFileManager().publish(container.queryJob(), outputName, *patchDesc, &patchFile);
  161. try { // set file size
  162. if (patchFile) {
  163. __int64 fs = patchFile->getFileSize(true,false);
  164. if (fs!=-1)
  165. patchFile->queryAttributes().setPropInt64("@size",fs);
  166. }
  167. }
  168. catch (IException *e) {
  169. EXCLOG(e,"keydiff setting file size");
  170. e->Release();
  171. }
  172. // Add a new 'Patch' description to the secondary key.
  173. DistributedFilePropertyLock lock(newIndexFile);
  174. IPropertyTree &fileProps = lock.queryAttributes();
  175. StringBuffer path("Patch[@name=\"");
  176. path.append(outputName).append("\"]");
  177. IPropertyTree *patch = fileProps.queryPropTree(path.str());
  178. if (!patch) patch = fileProps.addPropTree("Patch", createPTree());
  179. patch->setProp("@name", outputName);
  180. unsigned checkSum;
  181. if (patchFile->getFileCheckSum(checkSum))
  182. patch->setPropInt64("@checkSum", checkSum);
  183. IPropertyTree *index = patch->setPropTree("Index", createPTree());
  184. index->setProp("@name", originalIndexFile->queryLogicalName());
  185. if (originalIndexFile->getFileCheckSum(checkSum))
  186. index->setPropInt64("@checkSum", checkSum);
  187. CMasterActivity::done();
  188. }
  189. virtual void preStart(size32_t parentExtractSz, const byte *parentExtract)
  190. {
  191. CMasterActivity::preStart(parentExtractSz, parentExtract);
  192. IHThorKeyDiffArg *helper = (IHThorKeyDiffArg *) queryHelper();
  193. if (0==(KDPoverwrite & helper->getFlags()))
  194. {
  195. if (KDPvaroutputname & helper->getFlags()) return;
  196. OwnedRoxieString outputHelperName(helper->getOutputName());
  197. Owned<IDistributedFile> file = queryThorFileManager().lookup(container.queryJob(), outputHelperName, false, true);
  198. if (file)
  199. throw MakeActivityException(this, TE_OverwriteNotSpecified, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", file->queryLogicalName());
  200. }
  201. }
  202. virtual void kill()
  203. {
  204. CMasterActivity::kill();
  205. originalIndexFile.clear();
  206. newIndexFile.clear();
  207. }
  208. };
  209. CActivityBase *createKeyDiffActivityMaster(CMasterGraphElement *container)
  210. {
  211. return new CKeyDiffMaster(container);
  212. }