Merge pull request #63 from vmarkovtsev/master

Tensorboard and labours.py fixes

labours.py

@@ -195,6 +195,8 @@ class YamlReader(Reader):
         # turn strings into ints
         for item in obj:
             item.counters = {int(k): v for k, v in item.counters.items()}
+        if len(obj) == 0:
+            raise KeyError
         return obj
 
     def _parse_burndown_matrix(self, matrix):
@@ -293,9 +295,11 @@ class ProtobufReader(Reader):
         from scipy.sparse import csr_matrix
         return index, csr_matrix((data, indices, indptr), shape=(len(shotness),) * 2)
 
-
     def get_shotness(self):
-        return self.contents["Shotness"].records
+        records = self.contents["Shotness"].records
+        if len(records) == 0:
+            raise KeyError
+        return records
 
     def _parse_burndown_matrix(self, matrix):
         dense = numpy.zeros((matrix.number_of_rows, matrix.number_of_columns), dtype=int)
@@ -1045,7 +1049,7 @@ def main():
         "--burndown --burndown-people."
     couples_warning = "Coupling stats were not collected. Re-run hercules with --couples."
     shotness_warning = "Structural hotness stats were not collected. Re-run hercules with " \
-                       "--shotness."
+                       "--shotness. Also check --languages - the output may be empty."
 
     def project_burndown():
         try:

shotness.go

@@ -428,6 +428,9 @@ func (shotness *ShotnessAnalysis) extractNodes(root *uast.Node) (map[string]*uas
 	// otherwise due to UAST quirks there may be false positives
 	internal := map[*uast.Node]bool{}
 	for _, mainNode := range structs {
+		if internal[mainNode] {
+			continue
+		}
 		subs, err := tools.Filter(mainNode, shotness.XpathStruct)
 		if err != nil {
 			return nil, err

swivel.py

@@ -72,10 +72,10 @@ flags.DEFINE_string("output_base_path", None,
 flags.DEFINE_integer("embedding_size", 300, "Size of the embeddings")
 flags.DEFINE_boolean("trainable_bias", False, "Biases are trainable")
 flags.DEFINE_integer("submatrix_rows", 4096,
-                     "Rows in each training submatrix. This must match"
+                     "Rows in each training submatrix. This must match "
                      "the training data.")
 flags.DEFINE_integer("submatrix_cols", 4096,
-                     "Rows in each training submatrix. This must match"
+                     "Rows in each training submatrix. This must match "
                      "the training data.")
 flags.DEFINE_float("loss_multiplier", 1.0 / 4096,
                    "constant multiplier on loss.")
@@ -205,9 +205,9 @@ class SwivelModel:
 
         # Create paths to input data files
         log("Reading model from: %s", config.input_base_path)
-        count_matrix_files = glob.glob(config.input_base_path + "/shard-*.pb")
-        row_sums_path = config.input_base_path + "/row_sums.txt"
-        col_sums_path = config.input_base_path + "/col_sums.txt"
+        count_matrix_files = glob.glob(os.path.join(config.input_base_path, "shard-*.pb"))
+        row_sums_path = os.path.join(config.input_base_path, "row_sums.txt")
+        col_sums_path = os.path.join(config.input_base_path, "col_sums.txt")
 
         # Read marginals
         row_sums = read_marginals_file(row_sums_path)
@@ -367,6 +367,8 @@ class SwivelModel:
             tf.zeros((length, self._config.embedding_size)),
             name="top10k_embedding")
         embedding_config.tensor_name = self.embedding10k.name
+        embedding_config.metadata_path = os.path.join(
+            self._config.input_base_path, "row_vocab.txt")
         tf.contrib.tensorboard.plugins.projector.visualize_embeddings(
             self.writer, projector_config)
         self.saver = tf.train.Saver((self.embedding10k,), max_to_keep=1)

uast_test.go

@@ -124,7 +124,7 @@ func TestUASTExtractorConsume(t *testing.T) {
 	// Language not enabled
 	assert.Len(t, res[DependencyUasts], 0)
 	assert.Nil(t, err)
-	exr.Languages["Go"] = true
+	exr.Languages["Go3000"] = true
 	res, err = exr.Consume(deps)
 	// No Go driver
 	assert.Len(t, res[DependencyUasts], 0)