converted lambdas to loops in consume_tile_traffic

tilezen · May 9, 2017 · e95e234 · e95e234
1 parent 5281f80
commit e95e234
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 15 deletions.
diff --git a/tilequeue/command.py b/tilequeue/command.py
@@ -956,8 +956,7 @@ def tilequeue_enqueue_tiles_of_interest(cfg, peripherals):
 def tilequeue_consume_tile_traffic(cfg, peripherals):
     logger = make_logger(cfg, 'consume_tile_traffic')
     logger.info('Consuming tile traffic logs ...')
-    logger.info(cfg.tile_traffic_log_path)
-
+
     iped_dated_coords = None
     with open(cfg.tile_traffic_log_path, 'r') as log_file:
         iped_dated_coords = parse_log_file(log_file)
@@ -977,13 +976,16 @@ def tilequeue_consume_tile_traffic(cfg, peripherals):
         # insert the log records after the latest_date
         cursor.execute('SELECT max(date) from tile_traffic_v4')
         max_timestamp = cursor.fetchone()[0]
-        iped_dated_coords_to_insert = filter(lambda iped_dated_coord: iped_dated_coord[1] > max_timestamp, iped_dated_coords) if max_timestamp else iped_dated_coords
-        for (host, timestamp, marchalled_coord) in iped_dated_coords_to_insert:
-            coord = coord_unmarshall_int(marchalled_coord)
-            cursor.execute("INSERT into tile_traffic_v4 (date, z, x, y, tilesize, service, host) VALUES ('%s', %d, %d, %d, %d, '%s', '%s')"
+
+        n_coords_inserted = 0
+        for host, timestamp, coord_int in iped_dated_coords:
+            if not max_timestamp or timestamp > max_timestamp:
+                coord = coord_unmarshall_int(coord_int)
+                cursor.execute("INSERT into tile_traffic_v4 (date, z, x, y, tilesize, service, host) VALUES ('%s', %d, %d, %d, %d, '%s', '%s')"
                             % (timestamp, coord.zoom, coord.column, coord.row, 512, 'vector-tiles', host))
+                n_coords_inserted += 1
 
-        logger.info('Inserted %d records' % len(iped_dated_coords_to_insert))
+        logger.info('Inserted %d records' % n_coords_inserted)
 
     sql_conn_pool.put_conns([sql_conn])
 

diff --git a/tilequeue/utils.py b/tilequeue/utils.py
@@ -34,15 +34,15 @@ def parse_log_file(log_file):
     tile_id_pattern = '\/([\w]+)\/([\d]+)\/([\d]+)\/([\d]+)\.([\d\w]*)'
 
     log_pattern = '%s - - %s "([\w]+) %s.*' % (ip_pattern, date_pattern, tile_id_pattern)
-
-    matches = filter(
-        lambda match: match and len(match.groups()) == 8, 
-        map(lambda log_string: re.search(log_pattern, log_string), log_file)
-    )
 
-    iped_dated_coords = map(lambda match: (match.group(1), 
-                                           datetime.strptime(match.group(2), '%d/%B/%Y %H:%M:%S'), 
-                                           coord_marshall_int(create_coord(match.group(6), match.group(7), match.group(5)))), matches)
+    iped_dated_coords = []
+    for log_string in log_file:
+        match = re.search(log_pattern, log_string)
+        if match and len(match.groups()) == 8:
+            iped_dated_coords.append((match.group(1), 
+                                      datetime.strptime(match.group(2), '%d/%B/%Y %H:%M:%S'), 
+                                      coord_marshall_int(create_coord(match.group(6), match.group(7), match.group(5)))))
+
     return iped_dated_coords
 
 def mimic_prune_tiles_of_interest_sql_structure(cursor):