Added *VehicleTurningRoute* variations

leaderboard/leaderboard_evaluator.py

@@ -52,8 +52,6 @@ class LeaderboardEvaluator(object):
     TODO: document me!
     """
 
-    ego_vehicles = []
-
     # Tunable parameters
     client_timeout = 10.0  # in seconds
     wait_for_world = 20.0  # in seconds
@@ -128,12 +126,6 @@ def _cleanup(self):
         """
         CarlaDataProvider.cleanup()
 
-        for i, _ in enumerate(self.ego_vehicles):
-            if self.ego_vehicles[i]:
-                self.ego_vehicles[i].destroy()
-                self.ego_vehicles[i] = None
-        self.ego_vehicles = []
-
         if self._agent_watchdog:
             self._agent_watchdog.stop()
 
@@ -148,6 +140,7 @@ def _cleanup(self):
         if self.manager and self.manager.get_running_status() \
                 and hasattr(self, 'world') and self.world:
             # Reset to asynchronous mode
+            self.world.tick()  # TODO: Understand why without this, an actor destruction runtime error occurs
             settings = self.world.get_settings()
             settings.synchronous_mode = False
             settings.fixed_delta_seconds = None

leaderboard/scenarios/route_scenario.py

@@ -66,7 +66,9 @@ def __init__(self, world, config, debug_mode=0, criteria_enable=True):
         self.route = self._get_route(config)
         sampled_scenario_definitions = self._filter_scenarios(config.scenario_configs)
 
-        ego_vehicle = self._spawn_ego_vehicle()
+        ego_vehicle = self._spawn_ego_vehicle(world)
+        if ego_vehicle is None:
+            raise ValueError("Shutting down, couldn't spawn the ego vehicle")
         self.timeout = self._estimate_route_timeout()
 
         if debug_mode>0:
@@ -114,21 +116,23 @@ def _filter_scenarios(self, scenario_configs):
 
         return new_scenarios_config
 
-    def _spawn_ego_vehicle(self):
+    def _spawn_ego_vehicle(self, world):
         """Spawn the ego vehicle at the first waypoint of the route"""
         elevate_transform = self.route[0][0]
         elevate_transform.location.z += 0.5
 
-        ego_vehicle = CarlaDataProvider.request_new_actor('vehicle.lincoln.mkz_2017',
+        ego_vehicle = CarlaDataProvider.request_new_actor('vehicle.lincoln.mkz_2020',
                                                           elevate_transform,
                                                           rolename='hero')
-
+        if not ego_vehicle:
+            return
 
         spectator = CarlaDataProvider.get_world().get_spectator()
-        ego_trans = ego_vehicle.get_transform()
-        spectator.set_transform(carla.Transform(ego_trans.location + carla.Location(z=50),
+        spectator.set_transform(carla.Transform(elevate_transform.location + carla.Location(z=50),
                                                     carla.Rotation(pitch=-90)))
 
+        world.tick()
+
         return ego_vehicle
 
     def _estimate_route_timeout(self):

leaderboard/scenarios/scenario_manager.py

@@ -175,7 +175,8 @@ def _tick_scenario(self):
             self._watchdog.resume()
             self.ego_vehicles[0].apply_control(ego_action)
 
-            # Tick scenario
+            # Tick scenario. Add the ego control to the blackboard in case some behaviors want to change it
+            py_trees.blackboard.Blackboard().set("AV_control", ego_action, overwrite=True)
             self.scenario_tree.tick_once()
 
             if self._debug_mode > 1:
@@ -191,7 +192,8 @@ def _tick_scenario(self):
                 self._statistics_manager.write_live_results(
                     self.config.index,
                     self.ego_vehicles[0].get_velocity().length(),
-                    ego_action
+                    ego_action,
+                    self.ego_vehicles[0].get_location()
                 )
 
             if self._debug_mode > 2:
@@ -204,7 +206,7 @@ def _tick_scenario(self):
                 self._running = False
 
             ego_trans = self.ego_vehicles[0].get_transform()
-            self._spectator.set_transform(carla.Transform(ego_trans.location + carla.Location(z=80),
+            self._spectator.set_transform(carla.Transform(ego_trans.location + carla.Location(z=70),
                                                           carla.Rotation(pitch=-90)))
 
     def get_running_status(self):

leaderboard/utils/result_writer.py

@@ -82,25 +82,14 @@ def create_output_text(self):
         list_statistics = [header]
         criteria_data = OrderedDict()
 
-        scenario_times = {}
         for criterion in self._data.scenario.get_criteria():
 
-            # Ignore the value of the YieldToEV test is not initialized.
             name = criterion.name
-            if name == "YieldToEmergencyVehicleTest" and not criterion.initialized:
-                continue
 
-            # If two criterion have the same name, their results are shown as one.
-            # Criteria with a "%" based value get their mean shown, while the rest show the sum.
             if name in criteria_data:
                 result = criterion.test_status
                 if STATUS_PRIORITY[result] < STATUS_PRIORITY[criteria_data[name]['result']]:
                     criteria_data[name]['result'] = result
-                if criterion.units == "%":
-                    if name in scenario_times:
-                        scenario_times[name] += 1
-                    else:
-                        scenario_times[name] = 1
                 criteria_data[name]['actual_value'] += criterion.actual_value
 
             else:
@@ -110,11 +99,6 @@ def create_output_text(self):
                     'units': criterion.units
                 }
 
-        # Get the mean value
-        for name, times in list(scenario_times.items()):
-            criteria_data[name]['actual_value'] /= times
-            criteria_data[name]['actual_value'] = round(criteria_data[name]['actual_value'], 2)
-
         for criterion_name in criteria_data:
             criterion = criteria_data[criterion_name]
 

leaderboard/utils/statistics_manager.py

@@ -26,14 +26,14 @@
     TrafficEventType.TRAFFIC_LIGHT_INFRACTION: 0.7,
     TrafficEventType.STOP_INFRACTION: 0.8,
     TrafficEventType.SCENARIO_TIMEOUT: 0.7,
+    TrafficEventType.YIELD_TO_EMERGENCY_VEHICLE: 0.7
 }
 PENALTY_PERC_DICT = {
     # Traffic events that substract a varying amount of points. This is the per unit value.
     # 'increases' means that the higher the value, the higher the penalty.
     # 'decreases' means that the ideal value is 100 and the lower the value, the higher the penalty.
     TrafficEventType.OUTSIDE_ROUTE_LANES_INFRACTION: [1, 'increases'],
     TrafficEventType.MIN_SPEED_INFRACTION: [0.2, 'decreases'],
-    TrafficEventType.YIELD_TO_EMERGENCY_VEHICLE: [0.25, 'decreases']
 }
 
 PENALTY_NAME_DICT = {
@@ -205,7 +205,7 @@ def sort_records(self):
             int(x.route_id.split('_rep')[-1])
         ))
 
-    def write_live_results(self, index, ego_speed, ego_control):
+    def write_live_results(self, index, ego_speed, ego_control, ego_location):
         """Writes live results"""
         route_record = self._results.checkpoint.records[index]
 
@@ -230,6 +230,7 @@ def write_live_results(self, index, ego_speed, ego_control):
                     "    Brake:              {:.3f}\n"
                     "    Steer:              {:.3f}\n\n"
                     "    Speed:           {:.3f} km/h\n\n"
+                    "    Location:           ({:.3f} {:.3f} {:.3f})\n\n"
                     "Total infractions: {}\n"
                     "Last 5 infractions:\n".format(
                         route_record.route_id,
@@ -243,6 +244,9 @@ def write_live_results(self, index, ego_speed, ego_control):
                         ego_control.brake,
                         ego_control.steer,
                         ego_speed * 3.6,
+                        ego_location.x,
+                        ego_location.y,
+                        ego_location.z,
                         route_record.num_infractions
                     )
                 )
@@ -389,6 +393,13 @@ def set_score_penalty(score_penalty):
 
     def compute_global_statistics(self):
         """Computes and saves the global statistics of the routes"""
+        def get_infractions_value(route_record, key):
+            # Special case for the % based criteria. Extract the meters from the message. Very ugly, but it works
+            if key != PENALTY_NAME_DICT[TrafficEventType.OUTSIDE_ROUTE_LANES_INFRACTION]:
+                return float(route_record.infractions[key][0].split(" ")[8])/1000
+
+            return len(route_record.infractions[key])
+
         global_record = GlobalRecord()
         global_result = global_record.result
 
@@ -401,12 +412,6 @@ def compute_global_statistics(self):
             global_record.scores_mean['Infraction penalty'] += route_record.scores['Infraction penalty'] / self._total_routes
             global_record.scores_mean['Driving score'] += route_record.scores['Driving score'] / self._total_routes
 
-            # Infractions
-            for key in global_record.infractions_per_km:
-                route_length = route_record.meta['Route length'] / 1000
-                route_completed = route_record.scores['Route completion'] / 100 * route_length
-                global_record.infractions_per_km[key] += len(route_record.infractions[key]) / max(route_completed, 0.001)
-
             # Downgrade the global result if need be ('Perfect' -> 'Completed' -> 'Failed'), and record the failed routes
             route_result = 'Failed' if 'Failed' in route_record.result else route_record.result 
             if route_result == 'Failed':
@@ -418,8 +423,18 @@ def compute_global_statistics(self):
         # Save the global result
         global_record.result = global_result
 
-        # Round the infractions number
+        # Calculate the number of infractions per km, and round the infractions number
+        km_driven = 0
+        for route_record in route_records:
+            km_driven += route_record.meta['Route length'] / 1000 * route_record.scores['Route completion'] / 100
+            for key in global_record.infractions_per_km:
+                global_record.infractions_per_km[key] += get_infractions_value(route_record, key)
+        km_driven = max(km_driven, 0.001)
+
         for key in global_record.infractions_per_km:
+            # Special case for the % based criteria.
+            if key != PENALTY_NAME_DICT[TrafficEventType.OUTSIDE_ROUTE_LANES_INFRACTION]:
+                global_record.infractions_per_km[key] /= km_driven
             global_record.infractions_per_km[key] = round(global_record.infractions_per_km[key], ROUND_DIGITS)
 
         # Scores standard deviation (Need the score mean to be calculated)

scripts/merge_statistics.py

@@ -49,7 +49,7 @@ def main():
     args = argparser.parse_args()
 
     # Initialize the statistics manager
-    statistics_manager = StatisticsManager(args.endpoint)
+    statistics_manager = StatisticsManager(args.endpoint, 0)
 
     # Make sure that the data is correctly formed
     sensors = []
@@ -82,7 +82,7 @@ def main():
     statistics_manager.save_sensors(sensors)
     statistics_manager.save_progress(total_routes, total_routes)
     statistics_manager.compute_global_statistics()
-    statistics_manager.validate_statistics()
+    statistics_manager.validate_and_write_statistics()
 
 
 if __name__ == '__main__':

scripts/scenario_creator.py

@@ -122,7 +122,7 @@
     ],
     "ConstructionObstacleTwoWays": [
         ["distance", "value"],
-        ["frequency", "value"],
+        ["frequency", "interval"],
     ],
     "Accident": [
         ["distance", "value"],
@@ -131,7 +131,7 @@
     ],
     "AccidentTwoWays": [
         ["distance", "value"],
-        ["frequency", "value"],
+        ["frequency", "interval"],
     ],
     "ParkedObstacle": [
         ["distance", "value"],
@@ -140,15 +140,15 @@
     ],
     "ParkedObstacleTwoWays": [
         ["distance", "value"],
-        ["frequency", "value"],
+        ["frequency", "interval"],
     ],
     "VehicleOpensDoor": [
         ["distance", "value"],
         ["speed", "value"],
     ],
     "VehicleOpensDoorTwoWays": [
         ["distance", "value"],
-        ["frequency", "value"],
+        ["frequency", "interval"],
     ],
     "HazardAtSideLane": [
         ["distance", "value"],
@@ -183,10 +183,6 @@
 
     # Others
     "BlockedIntersection": [
-        ["blocker_point", "location driving"],
-        ["obstacle_model", "value"],
-        ["obstacle_gap", "value"],
-        ["extra_obstacle", "value"],
     ],
     "InvadingTurn": [
         ["distance", "value"],