Add "shotness" presenter

Makefile

@@ -7,11 +7,14 @@ all: ${GOPATH}/bin/hercules
 test: all
 	go test gopkg.in/src-d/hercules.v3
 
-dependencies: ${GOPATH}/src/gopkg.in/bblfsh/client-go.v2 ${GOPATH}/src/gopkg.in/src-d/hercules.v3 ${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb.pb.go ${GOPATH}/src/gopkg.in/src-d/hercules.v3/cmd/hercules/plugin_template_source.go
+dependencies: ${GOPATH}/src/gopkg.in/bblfsh/client-go.v2 ${GOPATH}/src/gopkg.in/src-d/hercules.v3 ${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb.pb.go ${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb_pb2.py ${GOPATH}/src/gopkg.in/src-d/hercules.v3/cmd/hercules/plugin_template_source.go
 
-${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb.pb.go:
+${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb.pb.go: pb/pb.proto
 	PATH=$$PATH:$$GOPATH/bin protoc --gogo_out=pb --proto_path=pb pb/pb.proto
 
+${GOPATH}/src/gopkg.in/src-d/hercules.v3/pb/pb_pb2.py: pb/pb.proto
+	protoc --python_out pb --proto_path=pb pb/pb.proto
+
 ${GOPATH}/src/gopkg.in/src-d/hercules.v3/cmd/hercules/plugin_template_source.go: ${GOPATH}/src/gopkg.in/src-d/hercules.v3/cmd/hercules/plugin.template
 	cd ${GOPATH}/src/gopkg.in/src-d/hercules.v3/cmd/hercules && go generate
 

README.md

@@ -180,6 +180,32 @@ written to the current working directory with the name depending on `-o`. The ou
 and matches [Tensorflow Projector](http://projector.tensorflow.org/) so that the files and people
 can be visualized with t-SNE implemented in TF Projector.
 
+#### Structural hotness
+
+```
+       8  jinja2/runtime.py:__init__ [FunctionDef]
+       7  jinja2/ext.py:parse [FunctionDef]
+       6  jinja2/optimizer.py:optimize [FunctionDef]
+       6  jinja2/compiler.py:inspect [FunctionDef]
+       6  jinja2/loaders.py:__init__ [FunctionDef]
+       6  jinja2/exceptions.py:__init__ [FunctionDef]
+       6  jinja2/runtime.py:__getitem__ [FunctionDef]
+       5  jinja2/optimizer.py:visit_For [FunctionDef]
+       5  jinja2/parser.py:parse_tuple [FunctionDef]
+       5  jinja2/environment.py:compile [FunctionDef]
+```
+
+Thanks to Babelfish, hercules is able to measure how many times each structural unit has been modified.
+By default, it looks at functions; refer to [UAST XPath](https://doc.bblf.sh/user/uast-querying.html)
+manual to set an other query. **Beta**
+
+```
+hercules --shotness [--shotness-xpath-*]
+python3 labours.py -m shotness
+```
+
+Couples analysis automatically loads "shotness" data if available.
+
 #### Everything in a single pass
 
 ```

labours.py

@@ -30,6 +30,7 @@ if sys.version_info[0] < 3:
 PB_MESSAGES = {
     "Burndown": "pb.pb_pb2.BurndownAnalysisResults",
     "Couples": "pb.pb_pb2.CouplesAnalysisResults",
+    "Shotness": "pb.pb_pb2.ShotnessAnalysisResults",
 }
 
 
@@ -53,7 +54,7 @@ def parse_args():
     parser.add_argument("--couples-tmp-dir", help="Temporary directory to work with couples.")
     parser.add_argument("-m", "--mode",
                         choices=["project", "file", "person", "churn_matrix", "ownership",
-                                 "couples", "all"],
+                                 "couples", "shotness", "all"],
                         help="What to plot.")
     parser.add_argument(
         "--resample", default="year",
@@ -103,6 +104,9 @@ class Reader(object):
     def get_people_coocc(self):
         raise NotImplementedError
 
+    def get_shotness(self):
+        raise NotImplementedError
+
 
 class YamlReader(Reader):
     def read(self, file):
@@ -125,7 +129,7 @@ class YamlReader(Reader):
         self.data = data
 
     def get_name(self):
-        return next(iter(self.data["Burndown"]["project"]))
+        return self.data["hercules"]["repository"]
 
     def get_header(self):
         header = self.data["hercules"]
@@ -164,6 +168,14 @@ class YamlReader(Reader):
         coocc = self.data["Couples"]["people_coocc"]
         return coocc["index"], self._parse_coocc_matrix(coocc["matrix"])
 
+    def get_shotness(self):
+        from munch import munchify
+        obj = munchify(self.data["Shotness"])
+        # turn strings into ints
+        for item in obj:
+            item.counters = {int(k): v for k, v in item.counters.items()}
+        return obj
+
     def _parse_burndown_matrix(self, matrix):
         return numpy.array([numpy.fromstring(line, dtype=int, sep=" ")
                             for line in matrix.split("\n")])
@@ -183,7 +195,11 @@ class YamlReader(Reader):
 
 class ProtobufReader(Reader):
     def read(self, file):
-        from pb.pb_pb2 import AnalysisResults
+        try:
+            from pb.pb_pb2 import AnalysisResults
+        except ImportError as e:
+            print("\n\n>>> You need to generate pb/pb_pb2.py - run \"make\"\n", file=sys.stderr)
+            raise e from None
         self.data = AnalysisResults()
         if file != "-":
             with open(file, "rb") as fin:
@@ -238,6 +254,9 @@ class ProtobufReader(Reader):
         node = self.contents["Couples"].people_couples
         return list(node.index), self._parse_sparse_matrix(node.matrix)
 
+    def get_shotness(self):
+        return self.contents["Shotness"].records
+
     def _parse_burndown_matrix(self, matrix):
         dense = numpy.zeros((matrix.number_of_rows, matrix.number_of_columns), dtype=int)
         for y, row in enumerate(matrix.rows):
@@ -960,20 +979,29 @@ def write_embeddings(name, output, run_server, index, embeddings):
                 print("\t" + url)
 
 
+def show_shotness_stats(data):
+    top = sorted(((r.counters[i], i) for i, r in enumerate(data)), reverse=True)
+    for count, i in top:
+        r = data[i]
+        print("%8d  %s:%s [%s]" % (count, r.file, r.name, r.internal_role))
+
+
 def main():
     args = parse_args()
     reader = read_input(args)
     header = reader.get_header()
     name = reader.get_name()
 
-    burndown_warning = "Burndown stats were not collected. Re-run hercules with -burndown."
+    burndown_warning = "Burndown stats were not collected. Re-run hercules with --burndown."
     burndown_files_warning = \
         "Burndown stats for files were not collected. Re-run hercules with " \
-        "-burndown -burndown-files."
+        "--burndown --burndown-files."
     burndown_people_warning = \
         "Burndown stats for people were not collected. Re-run hercules with " \
-        "-burndown -burndown-people."
-    couples_warning = "Coupling stats were not collected. Re-run hercules with -couples."
+        "--burndown --burndown-people."
+    couples_warning = "Coupling stats were not collected. Re-run hercules with --couples."
+    shotness_warning = "Structural hotness stats were not collected. Re-run hercules with " \
+                       "--shotness."
 
     def project_burndown():
         try:
@@ -1037,6 +1065,14 @@ def main():
         except KeyError:
             print(couples_warning)
 
+    def shotness():
+        try:
+            data = reader.get_shotness()
+        except KeyError:
+            print(shotness_warning)
+            return
+        show_shotness_stats(data)
+
     if args.mode == "project":
         project_burndown()
     elif args.mode == "file":
@@ -1049,6 +1085,8 @@ def main():
         ownership_burndown()
     elif args.mode == "couples":
         couples()
+    elif args.mode == "shotness":
+        shotness()
     elif args.mode == "all":
         project_burndown()
         files_burndown()
@@ -1056,6 +1094,7 @@ def main():
         churn_matrix()
         ownership_burndown()
         couples()
+        shotness()
 
     if web_server.running:
         secs = int(os.getenv("COUPLES_SERVER_TIME", "60"))

pb/pb.proto

@@ -93,7 +93,7 @@ message ShotnessRecord {
     map<int32, int32> counters = 5;
 }
 
-message ShotnessAnalysisResultMessage {
+message ShotnessAnalysisResults {
     repeated ShotnessRecord records = 1;
 }
 

requirements.txt

@@ -11,3 +11,4 @@ PyYAML==3.12
 scipy==0.19.1
 six==1.10.0
 protobuf>=3.0.0
+munch>=2.0

shotness.go

@@ -47,8 +47,8 @@ type NodeSummary struct {
 	File         string
 }
 
-// ShotnessAnalysisResult is returned by Finalize() and represents the analysis result.
-type ShotnessAnalysisResult struct {
+// ShotnessResult is returned by Finalize() and represents the analysis result.
+type ShotnessResult struct {
 	Nodes    []NodeSummary
 	Counters []map[int]int
 }
@@ -279,7 +279,7 @@ func (shotness *ShotnessAnalysis) Consume(deps map[string]interface{}) (map[stri
 
 // Finalize produces the result of the analysis. No more Consume() calls are expected afterwards.
 func (shotness *ShotnessAnalysis) Finalize() interface{} {
-	result := ShotnessAnalysisResult{
+	result := ShotnessResult{
 		Nodes:    make([]NodeSummary, len(shotness.nodes)),
 		Counters: make([]map[int]int, len(shotness.nodes)),
 	}
@@ -309,7 +309,7 @@ func (shotness *ShotnessAnalysis) Finalize() interface{} {
 
 // Serialize converts the result from Finalize() to either Protocol Buffers or YAML.
 func (shotness *ShotnessAnalysis) Serialize(result interface{}, binary bool, writer io.Writer) error {
-	shotnessResult := result.(ShotnessAnalysisResult)
+	shotnessResult := result.(ShotnessResult)
 	if binary {
 		return shotness.serializeBinary(&shotnessResult, writer)
 	}
@@ -317,9 +317,9 @@ func (shotness *ShotnessAnalysis) Serialize(result interface{}, binary bool, wri
 	return nil
 }
 
-func (shotness *ShotnessAnalysis) serializeText(result *ShotnessAnalysisResult, writer io.Writer) {
+func (shotness *ShotnessAnalysis) serializeText(result *ShotnessResult, writer io.Writer) {
 	for i, summary := range result.Nodes {
-		fmt.Fprintf(writer, "  - name: %s\n    file: %s\n    ir: %s\n    roles: [",
+		fmt.Fprintf(writer, "  - name: %s\n    file: %s\n    internal_role: %s\n    roles: [",
 			summary.Name, summary.File, summary.InternalRole)
 		for j, r := range summary.Roles {
 			if j < len(summary.Roles)-1 {
@@ -339,17 +339,17 @@ func (shotness *ShotnessAnalysis) serializeText(result *ShotnessAnalysisResult,
 		for _, key := range keys {
 			val := result.Counters[i][key]
 			if j < len(result.Counters[i])-1 {
-				fmt.Fprintf(writer, "%d:%d,", key, val)
+				fmt.Fprintf(writer, "\"%d\":%d,", key, val)
 			} else {
-				fmt.Fprintf(writer, "%d:%d}\n", key, val)
+				fmt.Fprintf(writer, "\"%d\":%d}\n", key, val)
 			}
 			j++
 		}
 	}
 }
 
-func (shotness *ShotnessAnalysis) serializeBinary(result *ShotnessAnalysisResult, writer io.Writer) error {
-	message := pb.ShotnessAnalysisResultMessage{
+func (shotness *ShotnessAnalysis) serializeBinary(result *ShotnessResult, writer io.Writer) error {
+	message := pb.ShotnessAnalysisResults{
 		Records: make([]*pb.ShotnessRecord, len(result.Nodes)),
 	}
 	for i, summary := range result.Nodes {

shotness_test.go

@@ -57,7 +57,7 @@ func TestShotnessRegistration(t *testing.T) {
 	assert.Equal(t, tp.Elem().Name(), "ShotnessAnalysis")
 }
 
-func bakeShotness(t *testing.T) (*ShotnessAnalysis, ShotnessAnalysisResult) {
+func bakeShotness(t *testing.T) (*ShotnessAnalysis, ShotnessResult) {
 	sh := fixtureShotness()
 	bytes1, err := ioutil.ReadFile(path.Join("test_data", "1.java"))
 	assert.Nil(t, err)
@@ -102,7 +102,7 @@ func bakeShotness(t *testing.T) (*ShotnessAnalysis, ShotnessAnalysisResult) {
 	iresult, err = sh.Consume(state)
 	assert.Nil(t, err)
 	assert.Nil(t, iresult)
-	return sh, sh.Finalize().(ShotnessAnalysisResult)
+	return sh, sh.Finalize().(ShotnessResult)
 }
 
 func TestShotnessConsume(t *testing.T) {
@@ -170,7 +170,7 @@ func TestShotnessConsume(t *testing.T) {
 			assert.Len(t, node.Couples, 17)
 		}
 	}
-	result := sh.Finalize().(ShotnessAnalysisResult)
+	result := sh.Finalize().(ShotnessResult)
 	assert.Len(t, result.Nodes, 18)
 	assert.Len(t, result.Counters, 18)
 	assert.Equal(t, result.Nodes[14].String(),
@@ -199,94 +199,94 @@ func TestShotnessSerializeText(t *testing.T) {
 	sh.Serialize(result, false, buffer)
 	assert.Equal(t, buffer.String(), `  - name: testAddEntry
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testArchiveEquals
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testContainsAnyEntry
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testDuplicateEntryAtAddOrReplace
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testDuplicateEntryAtAdd
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testDuplicateEntryAtReplace
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testPackEntries
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testPackEntry
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testPreserveRoot
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testRemoveDirs
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testRemoveEntry
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testRepackArchive
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testUnexplode
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testUnpackEntryFromFile
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:2,14:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":2,"14":1,"15":1,"16":1,"17":1}
   - name: testUnpackEntryFromStreamToFile
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {13:1,14:1}
+    counters: {"13":1,"14":1}
   - name: testUnpackEntryFromStream
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: testZipException
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,59,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
   - name: unexplodeWithException
     file: test.java
-    ir: MethodDeclaration
+    internal_role: MethodDeclaration
     roles: [111,100,41,45]
-    counters: {0:1,1:1,2:1,3:1,4:1,5:1,6:1,7:1,8:1,9:1,10:1,11:1,12:1,13:1,15:1,16:1,17:1}
+    counters: {"0":1,"1":1,"2":1,"3":1,"4":1,"5":1,"6":1,"7":1,"8":1,"9":1,"10":1,"11":1,"12":1,"13":1,"15":1,"16":1,"17":1}
 `)
 }
 
@@ -294,7 +294,7 @@ func TestShotnessSerializeBinary(t *testing.T) {
 	sh, result := bakeShotness(t)
 	buffer := &bytes.Buffer{}
 	sh.Serialize(result, true, buffer)
-	message := pb.ShotnessAnalysisResultMessage{}
+	message := pb.ShotnessAnalysisResults{}
 	err := proto.Unmarshal(buffer.Bytes(), &message)
 	assert.Nil(t, err)
 	assert.Len(t, message.Records, 18)