Cover Burndown.addMatrix() with tests and pass them

burndown.go

@@ -78,11 +78,12 @@ type BurndownResult struct {
 }
 
 const (
-	ConfigBurndownGranularity = "Burndown.Granularity"
-	ConfigBurndownSampling    = "Burndown.Sampling"
-	ConfigBurndownTrackFiles  = "Burndown.TrackFiles"
-	ConfigBurndownTrackPeople = "Burndown.TrackPeople"
-	ConfigBurndownDebug       = "Burndown.Debug"
+	ConfigBurndownGranularity  = "Burndown.Granularity"
+	ConfigBurndownSampling     = "Burndown.Sampling"
+	ConfigBurndownTrackFiles   = "Burndown.TrackFiles"
+	ConfigBurndownTrackPeople  = "Burndown.TrackPeople"
+	ConfigBurndownDebug        = "Burndown.Debug"
+	DefaultBurndownGranularity = 30
 )
 
 func (analyser *BurndownAnalysis) Name() string {
@@ -104,12 +105,12 @@ func (analyser *BurndownAnalysis) ListConfigurationOptions() []ConfigurationOpti
 		Description: "How many days there are in a single band.",
 		Flag:        "granularity",
 		Type:        IntConfigurationOption,
-		Default:     30}, {
+		Default:     DefaultBurndownGranularity}, {
 		Name:        ConfigBurndownSampling,
 		Description: "How frequently to record the state in days.",
 		Flag:        "sampling",
 		Type:        IntConfigurationOption,
-		Default:     30}, {
+		Default:     DefaultBurndownGranularity}, {
 		Name:        ConfigBurndownTrackFiles,
 		Description: "Record detailed statistics per each file.",
 		Flag:        "burndown-files",
@@ -158,12 +159,19 @@ func (analyser *BurndownAnalysis) Flag() string {
 
 func (analyser *BurndownAnalysis) Initialize(repository *git.Repository) {
 	if analyser.Granularity <= 0 {
-		fmt.Fprintln(os.Stderr, "Warning: adjusted the granularity to 30 days")
-		analyser.Granularity = 30
+		fmt.Fprintf(os.Stderr, "Warning: adjusted the granularity to %d days\n",
+			DefaultBurndownGranularity)
+		analyser.Granularity = DefaultBurndownGranularity
 	}
 	if analyser.Sampling <= 0 {
-		fmt.Fprintln(os.Stderr, "Warning: adjusted the sampling to 30 days")
-		analyser.Sampling = 30
+		fmt.Fprintf(os.Stderr, "Warning: adjusted the sampling to %d days\n",
+			DefaultBurndownGranularity)
+		analyser.Sampling = DefaultBurndownGranularity
+	}
+	if analyser.Sampling > analyser.Granularity {
+		fmt.Fprintf(os.Stderr, "Warning: granularity may not be less than sampling, adjusted to %d\n",
+			analyser.Granularity)
+		analyser.Sampling = analyser.Granularity
 	}
 	analyser.repository = repository
 	analyser.globalStatus = map[int]int64{}
@@ -331,37 +339,67 @@ func (analyser *BurndownAnalysis) MergeResults(
 	for name, index := range people {
 		merged.reversedPeopleDict[index] = name
 	}
-
-	// interpolate to daily and sum
-	_ = bar1
-	_ = bar2
-	panic("not implemented")
-	// return merged
+	merged.GlobalHistory = mergeMatrices(
+		bar1.GlobalHistory, bar2.GlobalHistory,
+		bar1.granularity, bar1.sampling,
+		bar2.granularity, bar2.sampling,
+		c1, c2)
+	return merged
 }
 
 func mergeMatrices(m1, m2 [][]int64, granularity1, sampling1, granularity2, sampling2 int,
-	c1, c2 *CommonAnalysisResult) {
+	c1, c2 *CommonAnalysisResult) [][]int64 {
 	commonMerged := CommonAnalysisResult{}
 	commonMerged.Merge(c1)
 	commonMerged.Merge(c2)
-	size := (commonMerged.EndTime - commonMerged.BeginTime) / (3600 * 24)
-	daily := make([][]float32, size)
+
+	var granularity, sampling int
+	if sampling1 < sampling2 {
+		sampling = sampling1
+	} else {
+		sampling = sampling2
+	}
+	if granularity1 < granularity2 {
+		granularity = granularity1
+	} else {
+		granularity = granularity2
+	}
+
+	size := int((commonMerged.EndTime - commonMerged.BeginTime) / (3600 * 24))
+	daily := make([][]float32, size+granularity)
 	for i := range daily {
-		daily[i] = make([]float32, size)
+		daily[i] = make([]float32, size+sampling)
 	}
-	addMatrix(m1, granularity1, sampling1, daily,
+	addBurndownMatrix(m1, granularity1, sampling1, daily,
 		int(c1.BeginTime-commonMerged.BeginTime)/(3600*24))
-	addMatrix(m2, granularity2, sampling2, daily,
+	addBurndownMatrix(m2, granularity2, sampling2, daily,
 		int(c2.BeginTime-commonMerged.BeginTime)/(3600*24))
+
 	// convert daily to [][]int64
+	result := make([][]int64, (size+granularity-1)/granularity)
+	for i := range result {
+		result[i] = make([]int64, (size+sampling-1)/sampling)
+		for j := 0; j < len(result[i])*sampling; j += sampling {
+			if j >= size {
+				j = size - 1
+			}
+			accum := float32(0)
+			for k := i * granularity; k < (i+1)*granularity && k < len(result[i]); k++ {
+				accum += daily[k][j]
+			}
+			result[i][j] = int64(accum)
+		}
+	}
+	return result
 }
 
-func addMatrix(matrix [][]int64, granularity, sampling int, daily [][]float32, offset int) {
-	/*
-	 daily_matrix = numpy.zeros(
-	            (matrix.shape[0] * granularity, matrix.shape[1] * sampling),
-	            dtype=numpy.float32)
-	*/
+// Explode `matrix` so that it is daily sampled and has daily bands, shift by `offset` days
+// and add to the accumulator. `daily` size is square and is guaranteed to fit `matrix` by
+// the caller.
+// Rows: *at least* len(matrix) * sampling + offset
+// Columns: *at least* len(matrix[...]) * granularity + offset
+// `matrix` can be sparse, so that the last columns which are equal to 0 are truncated.
+func addBurndownMatrix(matrix [][]int64, granularity, sampling int, daily [][]float32, offset int) {
 	// Determine the maximum number of bands; the actual one may be larger but we do not care
 	maxCols := 0
 	for _, row := range matrix {
@@ -369,35 +407,142 @@ func addMatrix(matrix [][]int64, granularity, sampling int, daily [][]float32, o
 			maxCols = len(row)
 		}
 	}
-	// Ported from labours.py load_burndown()
-	for y := 0; y < maxCols; y++ {
-		for x := 0; x < len(matrix); x++ {
-			if (y+1)*granularity <= x*sampling {
-				// interpolate
-				var previous int64
-				if x > 0 && y < len(matrix[x-1]) {
-					previous = matrix[x-1][y]
+	neededRows := len(matrix)*sampling + offset
+	if len(daily) < neededRows {
+		panic(fmt.Sprintf("merge bug: too few daily rows: required %d, have %d",
+			neededRows, len(daily)))
+	}
+	if len(daily[0]) < maxCols {
+		panic(fmt.Sprintf("merge bug: too few daily cols: required %d, have %d",
+			maxCols, len(daily[0])))
+	}
+	for x := 0; x < maxCols; x++ {
+		for y := 0; y < len(matrix); y++ {
+			if x*granularity > (y+1)*sampling {
+				// the future is zeros
+				continue
+			}
+			decay := func(startIndex int, startVal float32) {
+				k := float32(matrix[y][x]) / startVal // <= 1
+				scale := float32((y+1)*sampling - startIndex)
+				for i := x * granularity; i < (x+1)*granularity; i++ {
+					initial := daily[startIndex-1+offset][i+offset]
+					for j := startIndex; j < (y+1)*sampling; j++ {
+						daily[j+offset][i+offset] = initial * (1 + (k-1)*float32(j-startIndex+1)/scale)
+					}
 				}
-				for i := 0; i < sampling; i++ {
-					var value float32
-					if y < len(matrix[x]) {
-						value = (float32(previous) +
-							float32((matrix[x][y]-previous)*int64(i))/float32(sampling)) / float32(granularity)
-					} else {
-						value = float32(previous) *
-							(float32(1) - float32(i)/float32(sampling)) / float32(granularity)
+			}
+			raise := func(finishIndex int, finishVal float32) {
+				var initial float32
+				if y > 0 {
+					initial = float32(matrix[y-1][x])
+				}
+				startIndex := y * sampling
+				if startIndex < x*granularity {
+					startIndex = x * granularity
+				}
+				avg := (finishVal - initial) / float32(finishIndex-startIndex)
+				for j := y * sampling; j < finishIndex; j++ {
+					for i := startIndex; i <= j; i++ {
+						daily[j+offset][i+offset] = avg
+					}
+				}
+				// copy [x*g..y*s)
+				for j := y * sampling; j < finishIndex; j++ {
+					for i := x * granularity; i < y*sampling; i++ {
+						daily[j+offset][i+offset] = daily[j-1+offset][i+offset]
 					}
-					for j := y * granularity; j < (y+1)*granularity; j++ {
-						daily[j+offset][x*sampling+i+offset] += value
+				}
+			}
+			if (x+1)*granularity >= (y+1)*sampling {
+				// x*granularity <= (y+1)*sampling
+				// 1. x*granularity <= y*sampling
+				//    y*sampling..(y+1)sampling
+				//
+				//       x+1
+				//        /
+				//       /
+				//      / y+1  -|
+				//     /        |
+				//    / y      -|
+				//   /
+				//  / x
+				//
+				// 2. x*granularity > y*sampling
+				//    x*granularity..(y+1)sampling
+				//
+				//       x+1
+				//        /
+				//       /
+				//      / y+1  -|
+				//     /        |
+				//    / x      -|
+				//   /
+				//  / y
+				if x*granularity <= y*sampling {
+					raise((y+1)*sampling, float32(matrix[y][x]))
+				} else {
+					raise((y+1)*sampling, float32(matrix[y][x]))
+					avg := float32(matrix[y][x]) / float32((y+1)*sampling-x*granularity)
+					for j := x * granularity; j < (y+1)*sampling; j++ {
+						for i := x * granularity; i <= j; i++ {
+							daily[j+offset][i+offset] = avg
+						}
+					}
+				}
+			} else if (x+1)*granularity >= y*sampling {
+				// y*sampling <= (x+1)*granularity < (y+1)sampling
+				// y*sampling..(x+1)*granularity
+				// (x+1)*granularity..(y+1)sampling
+				//        x+1
+				//         /\
+				//        /  \
+				//       /    \
+				//      /    y+1
+				//     /
+				//    y
+				v1 := float32(matrix[y-1][x])
+				v2 := float32(matrix[y][x])
+				var peak float32
+				delta := float32((x+1)*granularity - y*sampling)
+				var scale float32
+				var previous float32
+				if y > 0 && (y-1)*sampling >= x*granularity {
+					// x*g <= (y-1)*s <= y*s <= (x+1)*g <= (y+1)*s
+					//           |________|.......^
+					if y > 1 {
+						previous = float32(matrix[y-2][x])
+					}
+					scale = float32(sampling)
+				} else {
+					// (y-1)*s < x*g <= y*s <= (x+1)*g <= (y+1)*s
+					//            |______|.......^
+					if y == 0 {
+						scale = float32(sampling)
+					} else {
+						scale = float32(y*sampling - x*granularity)
 					}
 				}
-			} else if y*granularity <= (x+1)*sampling && y < len(matrix[x]) {
-				// fill constant
-				for suby := y*granularity + offset; suby < (y+1)*granularity+offset; suby++ {
-					for subx := suby; subx < (x+1)*sampling+offset; subx++ {
-						daily[suby][subx] += float32(matrix[x][y]) / float32(granularity)
+				peak = v1 + (v1-previous)/scale*delta
+				if v2 > peak {
+					// we need to adjust the peak, it may not be less than the decayed value
+					if y < len(matrix)-1 {
+						// y*s <= (x+1)*g <= (y+1)*s < (y+2)*s
+						//           ^.........|_________|
+						k := (v2 - float32(matrix[y+1][x])) / float32(sampling) // > 0
+						peak = float32(matrix[y][x]) + k*float32((y+1)*sampling-(x+1)*granularity)
+						// peak > v2 > v1
+					} else {
+						peak = v2
+						// not enough data to interpolate; this is at least not restricted
 					}
 				}
+				raise((x+1)*granularity, peak)
+				decay((x+1)*granularity, peak)
+			} else {
+				// (x+1)*granularity < y*sampling
+				// y*sampling..(y+1)sampling
+				decay(y*sampling, float32(matrix[y-1][x]))
 			}
 		}
 	}

burndown_test.go

@@ -73,6 +73,25 @@ func TestBurndownRegistration(t *testing.T) {
 	assert.Equal(t, tp.Elem().Name(), "BurndownAnalysis")
 }
 
+func TestBurndownInitialize(t *testing.T) {
+	burndown := BurndownAnalysis{}
+	burndown.Sampling = -10
+	burndown.Granularity = DefaultBurndownGranularity
+	burndown.Initialize(testRepository)
+	assert.Equal(t, burndown.Sampling, DefaultBurndownGranularity)
+	assert.Equal(t, burndown.Granularity, DefaultBurndownGranularity)
+	burndown.Sampling = 0
+	burndown.Granularity = DefaultBurndownGranularity - 1
+	burndown.Initialize(testRepository)
+	assert.Equal(t, burndown.Sampling, DefaultBurndownGranularity-1)
+	assert.Equal(t, burndown.Granularity, DefaultBurndownGranularity-1)
+	burndown.Sampling = DefaultBurndownGranularity - 1
+	burndown.Granularity = -10
+	burndown.Initialize(testRepository)
+	assert.Equal(t, burndown.Sampling, DefaultBurndownGranularity-1)
+	assert.Equal(t, burndown.Granularity, DefaultBurndownGranularity)
+}
+
 func TestBurndownConsumeFinalize(t *testing.T) {
 	burndown := BurndownAnalysis{
 		Granularity:  30,
@@ -525,3 +544,247 @@ func TestCheckClose(t *testing.T) {
 	closer := panickingCloser{}
 	assert.Panics(t, func() { checkClose(closer) })
 }
+
+func TestBurndownAddMatrix(t *testing.T) {
+	size := 5*3 + 1
+	daily := make([][]float32, size)
+	for i := range daily {
+		daily[i] = make([]float32, size)
+	}
+	added := make([][]int64, 5)
+	for i := range added {
+		added[i] = make([]int64, 3)
+		switch i {
+		case 0:
+			added[i][0] = 10
+		case 1:
+			added[i][0] = 18
+			added[i][1] = 2
+		case 2:
+			added[i][0] = 12
+			added[i][1] = 14
+		case 3:
+			added[i][0] = 10
+			added[i][1] = 12
+			added[i][2] = 6
+		case 4:
+			added[i][0] = 8
+			added[i][1] = 9
+			added[i][2] = 13
+		}
+	}
+	assert.Panics(t, func() {
+		daily2 := make([][]float32, 16)
+		for i := range daily2 {
+			daily2[i] = make([]float32, 15)
+		}
+		addBurndownMatrix(added, 5, 3, daily2, 1)
+	})
+	assert.Panics(t, func() {
+		daily2 := make([][]float32, 15)
+		for i := range daily2 {
+			daily2[i] = make([]float32, 16)
+		}
+		addBurndownMatrix(added, 5, 3, daily2, 1)
+	})
+	// yaml.PrintMatrix(os.Stdout, added, 0, "test", true)
+	/*
+		"test": |-
+	  10  0  0
+	  18  2  0
+	  12 14  0
+	  10 12  6
+	   8  9 13
+	*/
+	addBurndownMatrix(added, 5, 3, daily, 1)
+	for i := range daily[0] {
+		assert.Equal(t, daily[0][i], float32(0))
+	}
+	for i := range daily {
+		assert.Equal(t, daily[i][0], float32(0))
+	}
+	/*for _, row := range daily {
+		fmt.Println(row)
+	}*/
+	// check pinned points
+	for y := 0; y < 5; y++ {
+		for x := 0; x < 3; x++ {
+			var sum float32
+			for i := x * 5; i < (x+1)*5; i++ {
+				sum += daily[(y+1)*3][i+1]
+			}
+			assert.InDelta(t, sum, added[y][x], 0.00001)
+		}
+	}
+	// check overall trend: 0 -> const -> peak -> decay
+	for x := 0; x < 15; x++ {
+		for y := 0; y < x; y++ {
+			assert.Zero(t, daily[y+1][x+1])
+		}
+		var prev float32
+		for y := x; y < ((x+3)/5)*5; y++ {
+			if prev == 0 {
+				prev = daily[y+1][x+1]
+			}
+			assert.Equal(t, daily[y+1][x+1], prev)
+		}
+		for y := ((x + 3) / 5) * 5; y < 15; y++ {
+			if prev == 0 {
+				prev = daily[y+1][x+1]
+			}
+			assert.True(t, daily[y+1][x+1] <= prev)
+			prev = daily[y+1][x+1]
+		}
+	}
+}
+
+func TestBurndownAddMatrixCrazy(t *testing.T) {
+	size := 5 * 3
+	daily := make([][]float32, size)
+	for i := range daily {
+		daily[i] = make([]float32, size)
+	}
+	added := make([][]int64, 5)
+	for i := range added {
+		added[i] = make([]int64, 3)
+		switch i {
+		case 0:
+			added[i][0] = 10
+		case 1:
+			added[i][0] = 9
+			added[i][1] = 2
+		case 2:
+			added[i][0] = 8
+			added[i][1] = 16
+		case 3:
+			added[i][0] = 7
+			added[i][1] = 12
+			added[i][2] = 6
+		case 4:
+			added[i][0] = 6
+			added[i][1] = 9
+			added[i][2] = 13
+		}
+	}
+	// yaml.PrintMatrix(os.Stdout, added, 0, "test", true)
+	/*
+		"test": |-
+	  10  0  0
+	  9  2  0
+	  8 16  0
+	  7 12  6
+	  6  9 13
+	*/
+	addBurndownMatrix(added, 5, 3, daily, 0)
+	/*for _, row := range daily {
+		fmt.Println(row)
+	}*/
+	// check pinned points
+	for y := 0; y < 5; y++ {
+		for x := 0; x < 3; x++ {
+			var sum float32
+			for i := x * 5; i < (x+1)*5; i++ {
+				sum += daily[(y+1)*3-1][i]
+			}
+			assert.InDelta(t, sum, added[y][x], 0.00001)
+		}
+	}
+	// check overall trend: 0 -> const -> peak -> decay
+	for x := 0; x < 15; x++ {
+		for y := 0; y < x; y++ {
+			assert.Zero(t, daily[y][x])
+		}
+		var prev float32
+		for y := x; y < ((x+3)/5)*5; y++ {
+			if prev == 0 {
+				prev = daily[y][x]
+			}
+			assert.Equal(t, daily[y][x], prev)
+		}
+		for y := ((x + 3) / 5) * 5; y < 15; y++ {
+			if prev == 0 {
+				prev = daily[y][x]
+			}
+			assert.True(t, daily[y][x] <= prev)
+			prev = daily[y][x]
+		}
+	}
+}
+
+func TestBurndownMergeGlobalHistory(t *testing.T) {
+	people1 := [...]string{"one", "two"}
+	res1 := BurndownResult{
+		GlobalHistory:      [][]int64{},
+		FileHistories:      map[string][][]int64{},
+		PeopleHistories:    [][][]int64{},
+		PeopleMatrix:       [][]int64{},
+		reversedPeopleDict: people1[:],
+		sampling:           15,
+		granularity:        20,
+	}
+	c1 := CommonAnalysisResult{
+		BeginTime:     600566400, // 1989 Jan 12
+		EndTime:       604713600, // 1989 March 1
+		CommitsNumber: 10,
+		RunTime:       100000,
+	}
+	// 48 days
+	res1.GlobalHistory = make([][]int64, 48/15+1 /* 4 samples */)
+	for i := range res1.GlobalHistory {
+		res1.GlobalHistory[i] = make([]int64, 48/20+1 /* 3 bands */)
+		switch i {
+		case 0:
+			res1.GlobalHistory[i][0] = 1000
+		case 1:
+			res1.GlobalHistory[i][0] = 1100
+			res1.GlobalHistory[i][1] = 400
+		case 2:
+			res1.GlobalHistory[i][0] = 900
+			res1.GlobalHistory[i][1] = 750
+			res1.GlobalHistory[i][2] = 100
+		case 3:
+			res1.GlobalHistory[i][0] = 850
+			res1.GlobalHistory[i][1] = 700
+			res1.GlobalHistory[i][2] = 150
+		}
+	}
+	people2 := [...]string{"two", "three"}
+	res2 := BurndownResult{
+		GlobalHistory:      [][]int64{},
+		FileHistories:      map[string][][]int64{},
+		PeopleHistories:    [][][]int64{},
+		PeopleMatrix:       [][]int64{},
+		reversedPeopleDict: people2[:],
+		sampling:           14,
+		granularity:        19,
+	}
+	c2 := CommonAnalysisResult{
+		BeginTime:     601084800, // 1989 Jan 18
+		EndTime:       605923200, // 1989 March 15
+		CommitsNumber: 10,
+		RunTime:       100000,
+	}
+	// 56 days
+	res2.GlobalHistory = make([][]int64, 56/14 /* 4 samples */)
+	for i := range res2.GlobalHistory {
+		res2.GlobalHistory[i] = make([]int64, 56/19+1 /* 3 bands */)
+		switch i {
+		case 0:
+			res2.GlobalHistory[i][0] = 900
+		case 1:
+			res2.GlobalHistory[i][0] = 1100
+			res2.GlobalHistory[i][1] = 400
+		case 2:
+			res2.GlobalHistory[i][0] = 900
+			res2.GlobalHistory[i][1] = 750
+			res2.GlobalHistory[i][2] = 100
+		case 3:
+			res2.GlobalHistory[i][0] = 800
+			res2.GlobalHistory[i][1] = 600
+			res2.GlobalHistory[i][2] = 600
+		}
+	}
+	burndown := BurndownAnalysis{}
+	merged := burndown.MergeResults(res1, res2, &c1, &c2).(BurndownResult)
+	//fmt.Println(merged.granularity, merged.sampling, merged.GlobalHistory)
+}

labours.py

@@ -300,7 +300,7 @@ def load_burndown(header, name, matrix, resample):
                     value = ((previous + (matrix[y, x] - previous) * epsrange)
                              / granularity)[numpy.newaxis, :]
                     daily_matrix[y * granularity:(y + 1) * granularity,
-                    x * sampling:(x + 1) * sampling] = value
+                                 x * sampling:(x + 1) * sampling] = value
                 elif y * granularity <= (x + 1) * sampling:
                     for suby in range(y * granularity, (y + 1) * granularity):
                         for subx in range(suby, (x + 1) * sampling):