From e95e234d89c5531f46582ff3833743e31703061b Mon Sep 17 00:00:00 2001 From: ambientlight Date: Tue, 9 May 2017 15:59:01 +0800 Subject: [PATCH] converted lambdas to loops in consume_tile_traffic --- tilequeue/command.py | 16 +++++++++------- tilequeue/utils.py | 16 ++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tilequeue/command.py b/tilequeue/command.py index d0d04115..075c6d45 100755 --- a/tilequeue/command.py +++ b/tilequeue/command.py @@ -956,8 +956,7 @@ def tilequeue_enqueue_tiles_of_interest(cfg, peripherals): def tilequeue_consume_tile_traffic(cfg, peripherals): logger = make_logger(cfg, 'consume_tile_traffic') logger.info('Consuming tile traffic logs ...') - logger.info(cfg.tile_traffic_log_path) - + iped_dated_coords = None with open(cfg.tile_traffic_log_path, 'r') as log_file: iped_dated_coords = parse_log_file(log_file) @@ -977,13 +976,16 @@ def tilequeue_consume_tile_traffic(cfg, peripherals): # insert the log records after the latest_date cursor.execute('SELECT max(date) from tile_traffic_v4') max_timestamp = cursor.fetchone()[0] - iped_dated_coords_to_insert = filter(lambda iped_dated_coord: iped_dated_coord[1] > max_timestamp, iped_dated_coords) if max_timestamp else iped_dated_coords - for (host, timestamp, marchalled_coord) in iped_dated_coords_to_insert: - coord = coord_unmarshall_int(marchalled_coord) - cursor.execute("INSERT into tile_traffic_v4 (date, z, x, y, tilesize, service, host) VALUES ('%s', %d, %d, %d, %d, '%s', '%s')" + + n_coords_inserted = 0 + for host, timestamp, coord_int in iped_dated_coords: + if not max_timestamp or timestamp > max_timestamp: + coord = coord_unmarshall_int(coord_int) + cursor.execute("INSERT into tile_traffic_v4 (date, z, x, y, tilesize, service, host) VALUES ('%s', %d, %d, %d, %d, '%s', '%s')" % (timestamp, coord.zoom, coord.column, coord.row, 512, 'vector-tiles', host)) + n_coords_inserted += 1 - logger.info('Inserted %d records' % len(iped_dated_coords_to_insert)) + logger.info('Inserted %d records' % n_coords_inserted) sql_conn_pool.put_conns([sql_conn]) diff --git a/tilequeue/utils.py b/tilequeue/utils.py index a8a629ec..a14dbda1 100644 --- a/tilequeue/utils.py +++ b/tilequeue/utils.py @@ -34,15 +34,15 @@ def parse_log_file(log_file): tile_id_pattern = '\/([\w]+)\/([\d]+)\/([\d]+)\/([\d]+)\.([\d\w]*)' log_pattern = '%s - - %s "([\w]+) %s.*' % (ip_pattern, date_pattern, tile_id_pattern) - - matches = filter( - lambda match: match and len(match.groups()) == 8, - map(lambda log_string: re.search(log_pattern, log_string), log_file) - ) - iped_dated_coords = map(lambda match: (match.group(1), - datetime.strptime(match.group(2), '%d/%B/%Y %H:%M:%S'), - coord_marshall_int(create_coord(match.group(6), match.group(7), match.group(5)))), matches) + iped_dated_coords = [] + for log_string in log_file: + match = re.search(log_pattern, log_string) + if match and len(match.groups()) == 8: + iped_dated_coords.append((match.group(1), + datetime.strptime(match.group(2), '%d/%B/%Y %H:%M:%S'), + coord_marshall_int(create_coord(match.group(6), match.group(7), match.group(5))))) + return iped_dated_coords def mimic_prune_tiles_of_interest_sql_structure(cursor):