diff --git a/.gitignore b/.gitignore index 0d20b64..d051451 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,8 @@ *.pyc +*.json +*.json~ +.gitignore~ +.idea/ +3d/Hypos_A.txt~ +trackingResultsAPIDIS.xml + diff --git a/3d/Hypos_A.txt b/3d/Hypos_A.txt index a331c92..787ec91 100644 --- a/3d/Hypos_A.txt +++ b/3d/Hypos_A.txt @@ -13,7 +13,7 @@ 1113554107.026000 2 1685.494734 3541.761464 0.0 1113554108.026000 2 1729.33699 2962.741482 0.0 1113554109.026000 2 1970.671398 2849.704975 0.0 -1113554110.026000 2 2445.542949 2838.848469 0.0 3 573.746117 5186.383372 0.0 +1113554110.026000 2 2445.542949 2838.848469 0.0 3 573.746117 5186.383372 0.0 5 573.746117 5186.383372 0.0 7 573.746117 5186.383372 0.0 1113554111.026000 2 2834.758231 2653.298761 0.0 3 918.079426 5491.682199 0.0 1113554112.025000 2 2923.611116 2682.412139 0.0 3 1025.560041 5053.345676 0.0 1113554113.025000 2 3255.307079 3037.795496 0.0 3 1294.765459 4238.494895 0.0 diff --git a/formatchecker.py b/formatchecker.py index daccbbb..43d2f09 100644 --- a/formatchecker.py +++ b/formatchecker.py @@ -2,11 +2,12 @@ class FormatChecker: - def __init__(self, groundtruth, hypotheses): + def __init__(self, groundtruth, hypotheses, check3D = True): """Constructor """ self.groundtruth_ = groundtruth self.hypotheses_ = hypotheses + self.is3D = check3D def checkForAmbiguousIDs(self): @@ -78,7 +79,14 @@ def checkForCompleteness(self): """Check ground truth and hypotheses for containing width, height, x and y""" result = True - expectedKeys = ("x", "y", "width", "height") + + + if self.is3D : + expectedKeys = ("x", "y", "z") + else : + expectedKeys = ("x", "y", "width", "height") + + for f in self.groundtruth_["frames"]: for g in f["annotations"]: @@ -96,3 +104,4 @@ def checkForCompleteness(self): result &= False return result # true: OK, false: missing id found + diff --git a/groundtruth.json b/groundtruth.json index f713a65..1c2c90a 100644 --- a/groundtruth.json +++ b/groundtruth.json @@ -12,7 +12,8 @@ "width": 31.0, "id": "sheldon", "y": 105.0, - "x": 608.0 + "x": 608.0, + "z": 3.23 } ] }, @@ -27,7 +28,8 @@ "width": 31.0, "id": "sheldon", "y": 105.0, - "x": 608.0 + "x": 608.0, + "z": 3.23 }, { "dco": true, @@ -35,7 +37,9 @@ "width": 29.0, "id": "leonard", "y": 145.0, - "x": 622.0 + "x": 622.0, + "z": 3.23 + } ] } diff --git a/hypotheses.json b/hypotheses.json index 18acdb5..1612a54 100644 --- a/hypotheses.json +++ b/hypotheses.json @@ -11,7 +11,8 @@ "width": 31.0, "id": "sheldon", "y": 105.0, - "x": 608.0 + "x": 608.0, + "z": 3.23 } ] }, @@ -25,14 +26,16 @@ "width": 31.0, "id": "sheldon", "y": 105.0, - "x": 608.0 + "x": 608.0, + "z": 3.23 }, { "height": 38.0, "width": 29.0, "id": "leonard", "y": 145.0, - "x": 622.0 + "x": 622.0, + "z": 3.23 } ] } diff --git a/pymot.py b/pymot.py index 72a8de8..b6aa77b 100755 --- a/pymot.py +++ b/pymot.py @@ -3,6 +3,8 @@ import sys import json import argparse +import math + from munkres import Munkres from rect import Rect from importers import MOT_hypo_import @@ -15,16 +17,22 @@ class MOTEvaluation: - def __init__(self, groundtruth, hypotheses): + def __init__(self, groundtruth, hypotheses, use3Dinput=True): """Constructor """ - + + # is the input we use is 3D + self.use3D = use3Dinput + + self.distance_threshold = 50 # when using 3D input, the minimum distance between players should be 50 cm + self.overlap_threshold_ = 0.2 """Bounding box overlap threshold""" self.munkres_inf_ = sys.maxsize """Not quite infinite number for Munkres algorithm""" - - self.sync_delta_ = 0.001 + + # time is measure in days in the APIDIS dataset, not milliseconds + self.sync_delta_ = 1.0/24/60.0/60.0/100.0 # 2/100th of a second time difference from groundtruth #0.001 #measured in days, """Maximum offset considered for a match of hypothesis and ground truth""" self.groundtruth_ = groundtruth @@ -85,6 +93,7 @@ def evaluate(self): frames = self.groundtruth_["frames"] for frame in frames: + print "frame: ", frame["num"] self.evaluateFrame(frame) @@ -92,6 +101,7 @@ def evaluateFrame(self, frame): """Update statistics by evaluating a new frame.""" timestamp = frame["timestamp"] + LOG.info('timestamp %f', timestamp) groundtruths = frame["annotations"] hypotheses = self.get_hypotheses_frame(timestamp)["hypotheses"] @@ -133,6 +143,7 @@ def evaluateFrame(self, frame): # PAPER STEP 1 # Valid mappings skip Munkres algorithm, if both ground truth and hypo are found in this frame # We call these pairs correspondences and fill the list each frame. + # (python dictionary object) correspondences = {} # truth id -> hypothesis id listofprints = [] @@ -141,7 +152,7 @@ def evaluateFrame(self, frame): # print "DIFF Keep correspondence" for gt_id in self.mappings_.keys(): - groundtruth = filter(lambda g: g["id"] == gt_id, groundtruths) # Get ground truths with given ground truth id in current frame + groundtruth = filter(lambda g: g["id"] == gt_id, groundtruths) # Get ground truths with given ground truth id in current frame (is this necessary, if the formatcheck is done in advance?) if len(groundtruth) > 1: LOG.warning("found %d > 1 ground truth tracks for id %s", len(groundtruth), gt_id) elif len(groundtruth) < 1: @@ -153,14 +164,33 @@ def evaluateFrame(self, frame): continue # Hypothesis found for known mapping - # Check hypothesis for overlap - overlap = Rect(groundtruth[0]).overlap(Rect(hypothesis[0])) - if overlap >= self.overlap_threshold_: - LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"])) -# print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))) - listofprints.append("DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0])))) - correspondences[gt_id] = hypothesis[0]["id"] - self.total_overlap_ += overlap + + if self.use3D : + # Check hypothesis for overlap, for 3D data + x_gt = groundtruth[0]["x"] + y_gt = groundtruth[0]["y"] + x_hyp = hypothesis[0]["x"] + y_hyp = hypothesis[0]["y"] + + distance = math.sqrt((x_gt-x_hyp)**2 + (y_gt-y_hyp)**2) + + if distance <= self.distance_threshold: + LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"])) + # print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))) + listofprints.append("DIFF Keep corr %s %s %.2f" % ( + groundtruth[0]["id"], hypothesis[0]["id"], distance)) + correspondences[gt_id] = hypothesis[0]["id"] + self.total_euclidian_distance_error += distance + + else: + # Check hypothesis for overlap, for 2D data + overlap = Rect(groundtruth[0]).overlap(Rect(hypothesis[0])) + if overlap >= self.overlap_threshold_: + LOG.info("Keeping correspondence between %s and %s" % (groundtruth[0]["id"], hypothesis[0]["id"])) + # print "DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0]))) + listofprints.append("DIFF Keep corr %s %s %.2f" % (groundtruth[0]["id"], hypothesis[0]["id"], Rect(groundtruth[0]).overlap(Rect(hypothesis[0])))) + correspondences[gt_id] = hypothesis[0]["id"] + self.total_overlap_ += overlap for p in sorted(listofprints): @@ -190,16 +220,29 @@ def evaluateFrame(self, frame): if hypothesis["id"] in correspondences.values(): LOG.info("Hypothesis %s already in correspondence" % hypothesis["id"]) continue - - rect_groundtruth = Rect(groundtruth) - rect_hypothesis = Rect(hypothesis) - overlap = rect_groundtruth.overlap(rect_hypothesis) - - if overlap >= self.overlap_threshold_: -# print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap - munkres_matrix[i][j] = 1 / overlap - LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], overlap)) - + + if self.use3D: + x_gt = groundtruth["x"] + y_gt = groundtruth["y"] + x_hyp = hypothesis["x"] + y_hyp = hypothesis["y"] + + distance = math.sqrt((x_gt - x_hyp) ** 2 + (y_gt - y_hyp) ** 2) + + if distance <= self.distance_threshold: + # print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap + munkres_matrix[i][j] = distance + LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], distance)) + else: + rect_groundtruth = Rect(groundtruth) + rect_hypothesis = Rect(hypothesis) + overlap = rect_groundtruth.overlap(rect_hypothesis) + + if overlap >= self.overlap_threshold_: + # print "Fill Hungarian", rect_groundtruth, rect_hypothesis, overlap + munkres_matrix[i][j] = 1 / overlap + LOG.info("DIFF candidate %s %s %.2f" % (groundtruth["id"], hypothesis["id"], overlap)) + # Do the Munkres LOG.debug(munkres_matrix) @@ -227,20 +270,30 @@ def evaluateFrame(self, frame): # Assert no known mappings have been added to hungarian, since keep correspondence should have considered this case. if gt_id in self.mappings_: - assert self.mappings_[gt_id] != hypo_id - - + assert self.mappings_[gt_id] != hypo_id + + # Add to correspondences - LOG.info("Correspondence found: %s and %s (overlap: %f)" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index])) + if self.use3D: + LOG.info("Correspondence found: %s and %s (distance: %f)" % (gt_id, hypo_id, munkres_matrix[gt_index][hypo_index])) + else: + LOG.info("Correspondence found: %s and %s (overlap: %f)" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index])) # correspondencelist.append("DIFF correspondence %s %s %.2f" % (gt_id, hypo_id, 1.0 / munkres_matrix[gt_index][hypo_index])) correspondencelist.append("DIFF correspondence %s %s" % (gt_id, hypo_id)) correspondences[gt_id] = hypo_id - self.total_overlap_ += overlap - + + if self.use3D: + self.total_euclidian_distance_error += distance + else: + self.total_overlap_ += overlap + + # The dco flag stands for do not care and can be used to mark hard to track targets, e.g. because of occlusion. + # Thus, a tracker which does not find the target will not be penalized, whereas a tracker which finds the + # target won't be punished (with a false positive) either. # Count "recoverable" and "non-recoverable" mismatches # "recoverable" mismatches - if gt_id in self.gt_map_ and self.gt_map_[gt_id] != hypo_id and not groundtruths[gt_index].get("dco",False): + if gt_id in self.gt_map_ and self.gt_map_[gt_id] != hypo_id and not groundtruths[gt_index].get("dco",False): #get will return False by default if the key "dco" does not exist and its true value otherwise LOG.info("Look ma! We got a recoverable mismatch over here! (%s-%s) -> (%s-%s)" % (gt_id, self.gt_map_[gt_id], gt_id, hypo_id)) self.recoverable_mismatches_ += 1 @@ -423,7 +476,10 @@ def getMOTP(self): write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible") # raise("No correspondence found. MOTP calculation not possible") else: - motp = self.total_overlap_ / self.total_correspondences_ + if self.use3D: + motp = self.total_euclidian_distance_error / self.total_correspondences_ + else: + motp = self.total_overlap_ / self.total_correspondences_ return motp @@ -432,6 +488,16 @@ def getAbsoluteStatistics(self): covered_ground_truths = self.groundtruth_ids_ & set(self.gt_map_.keys()) lonely_hypotheses = self.hypothesis_ids_ - set(self.hypo_map_.keys()) + distanceOrOverlapString = "" + distanceValue = 0.0 + if self.use3D: + distanceOrOverlapString= "total distance error" + distanceValue = self.total_euclidian_distance_error + else: + distanceOrOverlapString = "total overlap" + distanceValue = self.total_overlap_ + + return { "ground truths": self.total_groundtruths_, "false positives": self.false_positives_, @@ -440,7 +506,7 @@ def getAbsoluteStatistics(self): "recoverable mismatches": self.recoverable_mismatches_, "non-recoverable mismatches": self.non_recoverable_mismatches_, "correspondences": self.total_correspondences_, - "total overlap": self.total_overlap_, + distanceOrOverlapString: distanceValue, "lonely ground truth tracks": len(lonely_ground_truths), "covered ground truth tracks": len(covered_ground_truths), "lonely hypothesis tracks": len(lonely_hypotheses), @@ -545,9 +611,16 @@ def resetStatistics(self): self.misses_ = 0 self.false_positives_ = 0 self.total_groundtruths_ = 0 - self.total_overlap_ = 0.0 + + if self.use3D: + self.total_euclidian_distance_error = 0.0 + else: + self.total_overlap_ = 0.0 + self.total_correspondences_ = 0 + # create two sets containing a unique collection of all the id that + # occur for every frame self.groundtruth_ids_ = set() self.hypothesis_ids_ = set() @@ -561,6 +634,7 @@ def resetStatistics(self): parser.add_argument('-v', '--visual_debug_file') args = parser.parse_args() + # Load ground truth according to format # Assume MOT format, if non-json gt = open(args.groundtruth) # gt file @@ -578,22 +652,24 @@ def resetStatistics(self): hypotheses = MOT_hypo_import(hypo.readlines()) hypo.close() - - evaluator = MOTEvaluation(groundtruth, hypotheses) - if(args.check_format): formatChecker = FormatChecker(groundtruth, hypotheses) success = formatChecker.checkForExistingIDs() - success |= formatChecker.checkForAmbiguousIDs() - success |= formatChecker.checkForCompleteness() + success &= formatChecker.checkForAmbiguousIDs() + success &= formatChecker.checkForCompleteness() if not success: write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n File: %s" % args.groundtruth) sys.exit() + # why would the evaluation start before checking format?? + evaluator = MOTEvaluation(groundtruth, hypotheses, True) + evaluator.evaluate() print "Track statistics" evaluator.printTrackStatistics() + + evaluator.getAbsoluteStatistics() print print "Results" evaluator.printResults()