forked from dataabc/weibo-crawler
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathweibo.py
executable file
·1701 lines (1565 loc) · 70.2 KB
/
weibo.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import codecs
import copy
import csv
import json
import logging
import logging.config
import math
import os
import random
import sqlite3
import sys
import warnings
import const
from collections import OrderedDict
from datetime import date, datetime, timedelta
from pathlib import Path
from time import sleep
import requests
from lxml import etree
from requests.adapters import HTTPAdapter
from tqdm import tqdm
from util import csvutil
warnings.filterwarnings("ignore")
# 如果日志文件夹不存在，则创建
if not os.path.isdir('log/'):
os.makedirs('log/')
logging_path = os.path.split(
os.path.realpath(__file__))[0] + os.sep + 'logging.conf'
logging.config.fileConfig(logging_path)
logger = logging.getLogger('weibo')
class Weibo(object):
def __init__(self, config):
"""Weibo类初始化"""
self.validate_config(config)
self.filter = config[
'filter'] # 取值范围为0、1,程序默认值为0,代表要爬取用户的全部微博,1代表只爬取用户的原创微博
self.remove_html_tag = config[
'remove_html_tag'] # 取值范围为0、1, 0代表不移除微博中的html tag, 1代表移除
since_date = config['since_date']
if isinstance(since_date, int):
since_date = date.today() - timedelta(since_date)
since_date = str(since_date)
self.since_date = since_date # 起始时间，即爬取发布日期从该值到现在的微博，形式为yyyy-mm-dd
self.start_page = config.get('start_page',
1) # 开始爬的页，如果中途被限制而结束可以用此定义开始页码
self.write_mode = config[
'write_mode'] # 结果信息保存类型，为list形式，可包含csv、mongo和mysql三种类型
self.original_pic_download = config[
'original_pic_download'] # 取值范围为0、1, 0代表不下载原创微博图片,1代表下载
self.retweet_pic_download = config[
'retweet_pic_download'] # 取值范围为0、1, 0代表不下载转发微博图片,1代表下载
self.original_video_download = config[
'original_video_download'] # 取值范围为0、1, 0代表不下载原创微博视频,1代表下载
self.retweet_video_download = config[
'retweet_video_download'] # 取值范围为0、1, 0代表不下载转发微博视频,1代表下载
self.download_comment = config['download_comment'] #1代表下载评论,0代表不下载
self.comment_max_download_count = config[
'comment_max_download_count'] #如果设置了下评论，每条微博评论数会限制在这个值内
self.result_dir_name = config.get(
'result_dir_name', 0) # 结果目录名，取值为0或1，决定结果文件存储在用户昵称文件夹里还是用户id文件夹里
cookie = config.get('cookie') # 微博cookie，可填可不填
user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'
self.headers = {'User_Agent': user_agent, 'Cookie': cookie}
self.mysql_config = config.get('mysql_config') # MySQL数据库连接配置，可以不填
user_id_list = config['user_id_list']
query_list = config.get('query_list') or []
if isinstance(query_list, str):
query_list = query_list.split(',')
self.query_list = query_list
if not isinstance(user_id_list, list):
if not os.path.isabs(user_id_list):
user_id_list = os.path.split(
os.path.realpath(__file__))[0] + os.sep + user_id_list
self.user_config_file_path = user_id_list # 用户配置文件路径
user_config_list = self.get_user_config_list(user_id_list)
else:
self.user_config_file_path = ''
user_config_list = [{
'user_id': user_id,
'since_date': self.since_date,
'query_list': query_list
} for user_id in user_id_list]
self.user_config_list = user_config_list # 要爬取的微博用户的user_config列表
self.user_config = {} # 用户配置,包含用户id和since_date
self.start_date = '' # 获取用户第一条微博时的日期
self.query = ''
self.user = {} # 存储目标微博用户信息
self.got_count = 0 # 存储爬取到的微博数
self.weibo = [] # 存储爬取到的所有微博信息
self.weibo_id_list = [] # 存储爬取到的所有微博id
def validate_config(self, config):
"""验证配置是否正确"""
# 验证filter、original_pic_download、retweet_pic_download、original_video_download、retweet_video_download
argument_list = [
'filter', 'original_pic_download', 'retweet_pic_download',
'original_video_download', 'retweet_video_download',
'download_comment'
]
for argument in argument_list:
if config[argument] != 0 and config[argument] != 1:
logger.warning(u'%s值应为0或1,请重新输入', config[argument])
sys.exit()
# 验证since_date
since_date = config['since_date']
if (not self.is_date(str(since_date))) and (not isinstance(
since_date, int)):
logger.warning(u'since_date值应为yyyy-mm-dd形式或整数,请重新输入')
sys.exit()
# 验证query_list
query_list = config.get('query_list') or []
if (not isinstance(query_list, list)) and (not isinstance(
query_list, str)):
logger.warning(u'query_list值应为list类型或字符串,请重新输入')
sys.exit()
# 验证write_mode
write_mode = ['csv', 'json', 'mongo', 'mysql', 'sqlite']
if not isinstance(config['write_mode'], list):
sys.exit(u'write_mode值应为list类型')
for mode in config['write_mode']:
if mode not in write_mode:
logger.warning(
u'%s为无效模式，请从csv、json、mongo和mysql中挑选一个或多个作为write_mode',
mode)
sys.exit()
# 验证运行模式
if('sqlite' not in config['write_mode'] and const.MODE == 'append'):
logger.warning(u'append模式下请将sqlite加入write_mode中')
sys.exit()
# 验证user_id_list
user_id_list = config['user_id_list']
if (not isinstance(user_id_list,
list)) and (not user_id_list.endswith('.txt')):
logger.warning(u'user_id_list值应为list类型或txt文件路径')
sys.exit()
if not isinstance(user_id_list, list):
if not os.path.isabs(user_id_list):
user_id_list = os.path.split(
os.path.realpath(__file__))[0] + os.sep + user_id_list
if not os.path.isfile(user_id_list):
logger.warning(u'不存在%s文件', user_id_list)
sys.exit()
comment_max_count = config['comment_max_download_count']
if (not isinstance(comment_max_count, int)):
logger.warning(u'最大下载评论数应为整数类型')
sys.exit()
elif (comment_max_count < 0):
logger.warning(u'最大下载数应该为正整数')
sys.exit()
def is_date(self, since_date):
"""判断日期格式是否正确"""
try:
datetime.strptime(since_date, '%Y-%m-%d')
return True
except ValueError:
return False
def get_json(self, params):
"""获取网页中json数据"""
url = 'https://m.weibo.cn/api/container/getIndex?'
r = requests.get(url,
params=params,
headers=self.headers,
verify=False)
return r.json(), r.status_code
def get_weibo_json(self, page):
"""获取网页中微博json数据"""
params = {
'container_ext': 'profile_uid:' + str(self.user_config['user_id']),
'containerid': '100103type=401&q=' + self.query,
'page_type': 'searchall'
} if self.query else {
'containerid': '107603' + str(self.user_config['user_id'])
}
params['page'] = page
js, _ = self.get_json(params)
return js
def user_to_csv(self):
"""将爬取到的用户信息写入csv文件"""
file_dir = os.path.split(
os.path.realpath(__file__))[0] + os.sep + 'weibo'
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
file_path = file_dir + os.sep + 'users.csv'
self.user_csv_file_path = file_path
result_headers = [
'用户id', '昵称', '性别', '生日', '所在地', '学习经历', '公司', '注册时间', '阳光信用',
'微博数', '粉丝数', '关注数', '简介', '主页', '头像', '高清头像', '微博等级', '会员等级',
'是否认证', '认证类型', '认证信息', '上次记录微博id'
]
result_data = [[
v.encode('utf-8') if 'unicode' in str(type(v)) else v
for v in self.user.values()
]]
# 已经插入信息的用户无需重复插入，返回的id是\n或者具体的id
self.last_weibo_id = csvutil.insert_or_update_user(logger, result_headers, result_data, file_path)
def user_to_mongodb(self):
"""将爬取的用户信息写入MongoDB数据库"""
user_list = [self.user]
self.info_to_mongodb('user', user_list)
logger.info(u'%s信息写入MongoDB数据库完毕', self.user['screen_name'])
def user_to_mysql(self):
"""将爬取的用户信息写入MySQL数据库"""
mysql_config = {
'host': 'localhost',
'port': 3306,
'user': 'root',
'password': '123456',
'charset': 'utf8mb4'
}
# 创建'weibo'数据库
create_database = """CREATE DATABASE IF NOT EXISTS weibo DEFAULT
CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"""
self.mysql_create_database(mysql_config, create_database)
# 创建'user'表
create_table = """
CREATE TABLE IF NOT EXISTS user (
id varchar(20) NOT NULL,
screen_name varchar(30),
gender varchar(10),
statuses_count INT,
followers_count INT,
follow_count INT,
registration_time varchar(20),
sunshine varchar(20),
birthday varchar(40),
location varchar(200),
education varchar(200),
company varchar(200),
description varchar(400),
profile_url varchar(200),
profile_image_url varchar(200),
avatar_hd varchar(200),
urank INT,
mbrank INT,
verified BOOLEAN DEFAULT 0,
verified_type INT,
verified_reason varchar(140),
PRIMARY KEY (id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4"""
self.mysql_create_table(mysql_config, create_table)
self.mysql_insert(mysql_config, 'user', [self.user])
logger.info(u'%s信息写入MySQL数据库完毕', self.user['screen_name'])
def user_to_database(self):
"""将用户信息写入文件/数据库"""
self.user_to_csv()
if 'mysql' in self.write_mode:
self.user_to_mysql()
if 'mongo' in self.write_mode:
self.user_to_mongodb()
if 'sqlite' in self.write_mode:
self.user_to_sqlite()
def get_user_info(self):
"""获取用户信息"""
params = {'containerid': '100505' + str(self.user_config['user_id'])}
# TODO 这里在读取下一个用户的时候很容易被ban，需要优化休眠时长
sleep(random.randint(6, 10))
js, status_code = self.get_json(params)
if status_code != 200:
logger.info(u"被ban了，需要等待一段时间")
sys.exit()
if js['ok']:
info = js['data']['userInfo']
user_info = OrderedDict()
user_info['id'] = self.user_config['user_id']
user_info['screen_name'] = info.get('screen_name', '')
user_info['gender'] = info.get('gender', '')
params = {
'containerid':
'230283' + str(self.user_config['user_id']) + '_-_INFO'
}
zh_list = [
u'生日', u'所在地', u'小学', u'初中', u'高中', u'大学', u'公司', u'注册时间',
u'阳光信用'
]
en_list = [
'birthday', 'location', 'education', 'education', 'education',
'education', 'company', 'registration_time', 'sunshine'
]
for i in en_list:
user_info[i] = ''
js, _ = self.get_json(params)
if js['ok']:
cards = js['data']['cards']
if isinstance(cards, list) and len(cards) > 1:
card_list = cards[0]['card_group'] + cards[1]['card_group']
for card in card_list:
if card.get('item_name') in zh_list:
user_info[en_list[zh_list.index(
card.get('item_name'))]] = card.get(
'item_content', '')
user_info['statuses_count'] = self.string_to_int(
info.get('statuses_count', 0))
user_info['followers_count'] = self.string_to_int(
info.get('followers_count', 0))
user_info['follow_count'] = self.string_to_int(
info.get('follow_count', 0))
user_info['description'] = info.get('description', '')
user_info['profile_url'] = info.get('profile_url', '')
user_info['profile_image_url'] = info.get('profile_image_url', '')
user_info['avatar_hd'] = info.get('avatar_hd', '')
user_info['urank'] = info.get('urank', 0)
user_info['mbrank'] = info.get('mbrank', 0)
user_info['verified'] = info.get('verified', False)
user_info['verified_type'] = info.get('verified_type', -1)
user_info['verified_reason'] = info.get('verified_reason', '')
user = self.standardize_info(user_info)
self.user = user
self.user_to_database()
return 0
else:
logger.info(u"user_id_list中 {} id出错".format(self.user_config['user_id']))
return -1
def get_long_weibo(self, id):
"""获取长微博"""
for i in range(5):
url = 'https://m.weibo.cn/detail/%s' % id
html = requests.get(url, headers=self.headers, verify=False).text
html = html[html.find('"status":'):]
html = html[:html.rfind('"hotScheme"')]
html = html[:html.rfind(',')]
html = '{' + html + '}'
js = json.loads(html, strict=False)
weibo_info = js.get('status')
if weibo_info:
weibo = self.parse_weibo(weibo_info)
return weibo
sleep(random.randint(6, 10))
def get_pics(self, weibo_info):
"""获取微博原始图片url"""
if weibo_info.get('pics'):
pic_info = weibo_info['pics']
pic_list = [pic['large']['url'] for pic in pic_info]
pics = ','.join(pic_list)
else:
pics = ''
return pics
def get_live_photo(self, weibo_info):
"""获取live photo中的视频url"""
live_photo_list = []
live_photo = weibo_info.get('pic_video')
if live_photo:
prefix = 'https://video.weibo.com/media/play?livephoto=//us.sinaimg.cn/'
for i in live_photo.split(','):
if len(i.split(':')) == 2:
url = prefix + i.split(':')[1] + '.mov'
live_photo_list.append(url)
return live_photo_list
def get_video_url(self, weibo_info):
"""获取微博视频url"""
video_url = ''
video_url_list = []
if weibo_info.get('page_info'):
if ((weibo_info['page_info'].get('urls')
or weibo_info['page_info'].get('media_info'))
and weibo_info['page_info'].get('type') == 'video'):
media_info = weibo_info['page_info']['urls']
if not media_info:
media_info = weibo_info['page_info']['media_info']
video_url = media_info.get('mp4_720p_mp4')
if not video_url:
video_url = media_info.get('mp4_hd_url')
if not video_url:
video_url = media_info.get('hevc_mp4_hd')
if not video_url:
video_url = media_info.get('mp4_sd_url')
if not video_url:
video_url = media_info.get('mp4_ld_mp4')
if not video_url:
video_url = media_info.get('stream_url_hd')
if not video_url:
video_url = media_info.get('stream_url')
if video_url:
video_url_list.append(video_url)
live_photo_list = self.get_live_photo(weibo_info)
if live_photo_list:
video_url_list += live_photo_list
return ';'.join(video_url_list)
def download_one_file(self, url, file_path, type, weibo_id):
"""下载单个文件(图片/视频)"""
try:
file_exist = os.path.isfile(file_path)
sqlite_exist = ('sqlite' in self.write_mode) and (
self.sqlite_exist_file(file_path))
need_download = (not file_exist) or (not sqlite_exist)
if need_download:
s = requests.Session()
s.mount(url, HTTPAdapter(max_retries=5))
flag = True
try_count = 0
while flag and try_count < 5:
flag = False
downloaded = s.get(url,
headers=self.headers,
timeout=(5, 10),
verify=False)
try_count += 1
if (url.endswith(('jpg', 'jpeg'))
and not downloaded.content.endswith(b'\xff\xd9')
) or (url.endswith('png') and
not downloaded.content.endswith(b'\xaeB`\x82')):
flag = True
#需要分别判断是否需要下载
if not file_exist:
with open(file_path, 'wb') as f:
f.write(downloaded.content)
if (not sqlite_exist) and ('sqlite' in self.write_mode):
self.insert_file_sqlite(file_path, weibo_id, url,
downloaded.content)
except Exception as e:
error_file = self.get_filepath(
type) + os.sep + 'not_downloaded.txt'
with open(error_file, 'ab') as f:
url = str(weibo_id) + ':' + file_path + ':' + url + '\n'
f.write(url.encode(sys.stdout.encoding))
logger.exception(e)
def sqlite_exist_file(self, url):
if not os.path.exists(self.get_sqlte_path()):
return True
con = self.get_sqlite_connection()
cur = con.cursor()
query_sql = """SELECT url FROM bins WHERE path=? """
count = cur.execute(query_sql, (url, )).fetchone()
con.close()
if count is None:
return False
return True
def insert_file_sqlite(self, file_path, weibo_id, url, binary):
if not weibo_id:
return
extension = Path(file_path).suffix
if not extension:
return
if len(binary) <= 0:
return
file_data = OrderedDict()
file_data["weibo_id"] = weibo_id
file_data["ext"] = extension
file_data["data"] = binary
file_data["path"] = file_path
file_data["url"] = url
con = self.get_sqlite_connection()
self.sqlite_insert(con, file_data, "bins")
con.close()
def handle_download(self, file_type, file_dir, urls, w):
"""处理下载相关操作"""
file_prefix = w['created_at'][:11].replace('-', '') + '_' + str(
w['id'])
if file_type == 'img':
if ',' in urls:
url_list = urls.split(',')
for i, url in enumerate(url_list):
index = url.rfind('.')
if len(url) - index >= 5:
file_suffix = '.jpg'
else:
file_suffix = url[index:]
file_name = file_prefix + '_' + str(i + 1) + file_suffix
file_path = file_dir + os.sep + file_name
self.download_one_file(url, file_path, file_type, w['id'])
else:
index = urls.rfind('.')
if len(urls) - index > 5:
file_suffix = '.jpg'
else:
file_suffix = urls[index:]
file_name = file_prefix + file_suffix
file_path = file_dir + os.sep + file_name
self.download_one_file(urls, file_path, file_type, w['id'])
else:
file_suffix = '.mp4'
if ';' in urls:
url_list = urls.split(';')
if url_list[0].endswith('.mov'):
file_suffix = '.mov'
for i, url in enumerate(url_list):
file_name = file_prefix + '_' + str(i + 1) + file_suffix
file_path = file_dir + os.sep + file_name
self.download_one_file(url, file_path, file_type, w['id'])
else:
if urls.endswith('.mov'):
file_suffix = '.mov'
file_name = file_prefix + file_suffix
file_path = file_dir + os.sep + file_name
self.download_one_file(urls, file_path, file_type, w['id'])
def download_files(self, file_type, weibo_type, wrote_count):
"""下载文件(图片/视频)"""
try:
describe = ''
if file_type == 'img':
describe = u'图片'
key = 'pics'
else:
describe = u'视频'
key = 'video_url'
if weibo_type == 'original':
describe = u'原创微博' + describe
else:
describe = u'转发微博' + describe
logger.info(u'即将进行%s下载', describe)
file_dir = self.get_filepath(file_type)
file_dir = file_dir + os.sep + describe
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
for w in tqdm(self.weibo[wrote_count:], desc='Download progress'):
if weibo_type == 'retweet':
if w.get('retweet'):
w = w['retweet']
else:
continue
if w.get(key):
self.handle_download(file_type, file_dir, w.get(key), w)
logger.info(u'%s下载完毕,保存路径:', describe)
logger.info(file_dir)
except Exception as e:
logger.exception(e)
def get_location(self, selector):
"""获取微博发布位置"""
location_icon = 'timeline_card_small_location_default.png'
span_list = selector.xpath('//span')
location = ''
for i, span in enumerate(span_list):
if span.xpath('img/@src'):
if location_icon in span.xpath('img/@src')[0]:
location = span_list[i + 1].xpath('string(.)')
break
return location
def get_article_url(self, selector):
"""获取微博中头条文章的url"""
article_url = ''
text = selector.xpath('string(.)')
if text.startswith(u'发布了头条文章'):
url = selector.xpath('//a/@data-url')
if url and url[0].startswith('http://t.cn'):
article_url = url[0]
return article_url
def get_topics(self, selector):
"""获取参与的微博话题"""
span_list = selector.xpath("//span[@class='surl-text']")
topics = ''
topic_list = []
for span in span_list:
text = span.xpath('string(.)')
if len(text) > 2 and text[0] == '#' and text[-1] == '#':
topic_list.append(text[1:-1])
if topic_list:
topics = ','.join(topic_list)
return topics
def get_at_users(self, selector):
"""获取@用户"""
a_list = selector.xpath('//a')
at_users = ''
at_list = []
for a in a_list:
if '@' + a.xpath('@href')[0][3:] == a.xpath('string(.)'):
at_list.append(a.xpath('string(.)')[1:])
if at_list:
at_users = ','.join(at_list)
return at_users
def string_to_int(self, string):
"""字符串转换为整数"""
if isinstance(string, int):
return string
elif string.endswith(u'万+'):
string = string[:-2] + '0000'
elif string.endswith(u'万'):
string = float(string[:-1]) * 10000
elif string.endswith(u'亿'):
string = float(string[:-1]) * 100000000
return int(string)
def standardize_date(self, created_at):
"""标准化微博发布时间"""
if u'刚刚' in created_at:
created_at = datetime.now().strftime('%Y-%m-%d')
elif u'分钟' in created_at:
minute = created_at[:created_at.find(u'分钟')]
minute = timedelta(minutes=int(minute))
created_at = (datetime.now() - minute).strftime('%Y-%m-%d')
elif u'小时' in created_at:
hour = created_at[:created_at.find(u'小时')]
hour = timedelta(hours=int(hour))
created_at = (datetime.now() - hour).strftime('%Y-%m-%d')
elif u'昨天' in created_at:
day = timedelta(days=1)
created_at = (datetime.now() - day).strftime('%Y-%m-%d')
else:
created_at = created_at.replace('+0800 ', '')
temp = datetime.strptime(created_at, '%c')
created_at = datetime.strftime(temp, '%Y-%m-%d')
return created_at
def standardize_info(self, weibo):
"""标准化信息，去除乱码"""
for k, v in weibo.items():
if 'bool' not in str(type(v)) and 'int' not in str(
type(v)) and 'list' not in str(
type(v)) and 'long' not in str(type(v)):
weibo[k] = v.replace(u'\u200b', '').encode(
sys.stdout.encoding, 'ignore').decode(sys.stdout.encoding)
return weibo
def parse_weibo(self, weibo_info):
weibo = OrderedDict()
if weibo_info['user']:
weibo['user_id'] = weibo_info['user']['id']
weibo['screen_name'] = weibo_info['user']['screen_name']
else:
weibo['user_id'] = ''
weibo['screen_name'] = ''
weibo['id'] = int(weibo_info['id'])
weibo['bid'] = weibo_info['bid']
text_body = weibo_info['text']
selector = etree.HTML(text_body)
if self.remove_html_tag:
weibo['text'] = selector.xpath('string(.)')
else:
weibo['text'] = text_body
weibo['article_url'] = self.get_article_url(selector)
weibo['pics'] = self.get_pics(weibo_info)
weibo['video_url'] = self.get_video_url(weibo_info)
weibo['location'] = self.get_location(selector)
weibo['created_at'] = weibo_info['created_at']
weibo['source'] = weibo_info['source']
weibo['attitudes_count'] = self.string_to_int(
weibo_info.get('attitudes_count', 0))
weibo['comments_count'] = self.string_to_int(
weibo_info.get('comments_count', 0))
weibo['reposts_count'] = self.string_to_int(
weibo_info.get('reposts_count', 0))
weibo['topics'] = self.get_topics(selector)
weibo['at_users'] = self.get_at_users(selector)
return self.standardize_info(weibo)
def print_user_info(self):
"""打印用户信息"""
logger.info('+' * 100)
logger.info(u'用户信息')
logger.info(u'用户id：%s', self.user['id'])
logger.info(u'用户昵称：%s', self.user['screen_name'])
gender = u'女' if self.user['gender'] == 'f' else u'男'
logger.info(u'性别：%s', gender)
logger.info(u'生日：%s', self.user['birthday'])
logger.info(u'所在地：%s', self.user['location'])
logger.info(u'教育经历：%s', self.user['education'])
logger.info(u'公司：%s', self.user['company'])
logger.info(u'阳光信用：%s', self.user['sunshine'])
logger.info(u'注册时间：%s', self.user['registration_time'])
logger.info(u'微博数：%d', self.user['statuses_count'])
logger.info(u'粉丝数：%d', self.user['followers_count'])
logger.info(u'关注数：%d', self.user['follow_count'])
logger.info(u'url：https://m.weibo.cn/profile/%s', self.user['id'])
if self.user.get('verified_reason'):
logger.info(self.user['verified_reason'])
logger.info(self.user['description'])
logger.info('+' * 100)
def print_one_weibo(self, weibo):
"""打印一条微博"""
try:
logger.info(u'微博id：%d', weibo['id'])
logger.info(u'微博正文：%s', weibo['text'])
logger.info(u'原始图片url：%s', weibo['pics'])
logger.info(u'微博位置：%s', weibo['location'])
logger.info(u'发布时间：%s', weibo['created_at'])
logger.info(u'发布工具：%s', weibo['source'])
logger.info(u'点赞数：%d', weibo['attitudes_count'])
logger.info(u'评论数：%d', weibo['comments_count'])
logger.info(u'转发数：%d', weibo['reposts_count'])
logger.info(u'话题：%s', weibo['topics'])
logger.info(u'@用户：%s', weibo['at_users'])
logger.info(u'url：https://m.weibo.cn/detail/%d', weibo['id'])
except OSError:
pass
def print_weibo(self, weibo):
"""打印微博，若为转发微博，会同时打印原创和转发部分"""
if weibo.get('retweet'):
logger.info('*' * 100)
logger.info(u'转发部分：')
self.print_one_weibo(weibo['retweet'])
logger.info('*' * 100)
logger.info(u'原创部分：')
self.print_one_weibo(weibo)
logger.info('-' * 120)
def get_one_weibo(self, info):
"""获取一条微博的全部信息"""
try:
weibo_info = info['mblog']
weibo_id = weibo_info['id']
retweeted_status = weibo_info.get('retweeted_status')
is_long = True if weibo_info.get(
'pic_num') > 9 else weibo_info.get('isLongText')
if retweeted_status and retweeted_status.get('id'): # 转发
retweet_id = retweeted_status.get('id')
is_long_retweet = retweeted_status.get('isLongText')
if is_long:
weibo = self.get_long_weibo(weibo_id)
if not weibo:
weibo = self.parse_weibo(weibo_info)
else:
weibo = self.parse_weibo(weibo_info)
if is_long_retweet:
retweet = self.get_long_weibo(retweet_id)
if not retweet:
retweet = self.parse_weibo(retweeted_status)
else:
retweet = self.parse_weibo(retweeted_status)
retweet['created_at'] = self.standardize_date(
retweeted_status['created_at'])
weibo['retweet'] = retweet
else: # 原创
if is_long:
weibo = self.get_long_weibo(weibo_id)
if not weibo:
weibo = self.parse_weibo(weibo_info)
else:
weibo = self.parse_weibo(weibo_info)
weibo['created_at'] = self.standardize_date(
weibo_info['created_at'])
return weibo
except Exception as e:
logger.exception(e)
def get_weibo_comments(self, weibo, max_count, on_downloaded):
"""
:weibo standardlized weibo
:max_count 最大允许下载数
:on_downloaded 下载完成时的实例方法回调
"""
if weibo['comments_count'] == 0:
return
logger.info(u'正在下载评论 微博id:{id}正文:{text}'.format(id=weibo['id'],
text=weibo['text']))
self._get_weibo_comments_cookie(weibo, 0, max_count, None,
on_downloaded)
def _get_weibo_comments_cookie(self, weibo, cur_count, max_count, max_id,
on_downloaded):
"""
:weibo standardlized weibo
:cur_count 已经下载的评论数
:max_count 最大允许下载数
:max_id 微博返回的max_id参数
:on_downloaded 下载完成时的实例方法回调
"""
if cur_count >= max_count:
return
id = weibo["id"]
params = {"mid": id}
if max_id:
params["max_id"] = max_id
url = "https://m.weibo.cn/comments/hotflow?max_id_type=0"
req = requests.get(
url,
params=params,
headers=self.headers,
)
json = None
error = False
try:
json = req.json()
except Exception as e:
#没有cookie会抓取失败
#微博日期小于某个日期的用这个url会被403 需要用老办法尝试一下
error = True
if error:
#最大好像只能有50条 TODO: improvement
self._get_weibo_comments_nocookie(weibo, 0, max_count, 0,
on_downloaded)
return
data = json.get('data')
if not data:
#新接口没有抓取到的老接口也试一下
self._get_weibo_comments_nocookie(weibo, 0, max_count, 0,
on_downloaded)
return
comments = data.get('data')
count = len(comments)
if count == 0:
#没有了可以直接跳出递归
return
if on_downloaded:
on_downloaded(weibo, comments)
#随机睡眠一下
if max_count % 40 == 0:
sleep(random.randint(1, 5))
cur_count += count
max_id = data.get('max_id')
if max_id == 0:
return
self._get_weibo_comments_cookie(weibo, cur_count, max_count, max_id,
on_downloaded)
def _get_weibo_comments_nocookie(self, weibo, cur_count, max_count, page,
on_downloaded):
"""
:weibo standardlized weibo
:cur_count 已经下载的评论数
:max_count 最大允许下载数
:max_id 微博返回的max_id参数
:on_downloaded 下载完成时的实例方法回调
"""
if cur_count >= max_count:
return
id = weibo['id']
url = "https://m.weibo.cn/api/comments/show?id={id}&page={page}".format(
id=id, page=page)
req = requests.get(url)
json = None
try:
json = req.json()
except Exception as e:
#没有cookie会抓取失败
logger.info(u'未能抓取评论 微博id:{id} 内容{text}'.format(
id=id, text=weibo['text']))
return
data = json.get('data')
if not data:
return
comments = data.get('data')
count = len(comments)
if count == 0:
#没有了可以直接跳出递归
return
if on_downloaded:
on_downloaded(weibo, comments)
cur_count += count
page += 1
#随机睡眠一下
if page % 2 == 0:
sleep(random.randint(1, 5))
req_page = data.get('max')
if req_page == 0:
return
if page >= req_page:
return
self._get_weibo_comments_nocookie(weibo, cur_count, max_count, page,
on_downloaded)
def is_pinned_weibo(self, info):
"""判断微博是否为置顶微博"""
weibo_info = info['mblog']
title = weibo_info.get('title')
if title and title.get('text') == u'置顶':
return True
else:
return False
def get_one_page(self, page):
"""获取一页的全部微博"""
try:
js = self.get_weibo_json(page)
if js['ok']:
weibos = js['data']['cards']
if self.query:
weibos = weibos[0]['card_group']
# 如果需要检查cookie，在循环第一个人的时候，就要看看仅自己可见的信息有没有，要是没有直接报错
for w in weibos:
if w['card_type'] == 9:
wb = self.get_one_weibo(w)
if wb:
if const.CHECK_COOKIE['CHECK'] and (not const.CHECK_COOKIE['CHECKED']) and wb['text'] == const.CHECK_COOKIE['HIDDEN_WEIBO'] + ' ':
const.CHECK_COOKIE['CHECKED'] = True
logger.info('cookie检查通过')
if const.CHECK_COOKIE['EXIT_AFTER_CHECK']:
return True
if wb['id'] in self.weibo_id_list:
continue
created_at = datetime.strptime(
wb['created_at'], '%Y-%m-%d')
since_date = datetime.strptime(
self.user_config['since_date'], '%Y-%m-%d')
if const.MODE == 'append':
# append模式下不会对置顶微博做任何处理
if self.is_pinned_weibo(w):
continue
if self.first_crawler and not self.is_pinned_weibo(w):
# 置顶微博的具体时间不好判定，将非置顶微博当成最新微博，写入上次抓取id的csv
self.latest_weibo_id = str(wb['id'])
csvutil.update_last_weibo_id(wb['user_id'], wb['id'], self.user_csv_file_path)
self.first_crawler = False
if str(wb['id']) == self.last_weibo_id:
if const.CHECK_COOKIE['CHECK'] and (not const.CHECK_COOKIE['CHECKED']):
# 已经爬取过最新的了，只是没检查到cookie，一旦检查通过，直接放行
const.CHECK_COOKIE['EXIT_AFTER_CHECK'] = True
continue
if self.last_weibo_id == self.latest_weibo_id:
logger.info('{} 用户没有发新微博'.format(self.user['screen_name']))
else:
logger.info('增量获取微博完毕，将最新微博id从 {} 变更为 {}'.format(self.last_weibo_id, self.latest_weibo_id))
return True
if created_at < since_date:
if self.is_pinned_weibo(w):
continue
else:
logger.info(
u'{}已获取{}({})的第{}页{}微博{}'.format(
'-' * 30, self.user['screen_name'],
self.user['id'], page,
'包含"' + self.query +
'"的' if self.query else '',
'-' * 30))
return True
if (not self.filter) or (
'retweet' not in wb.keys()):
self.weibo.append(wb)
self.weibo_id_list.append(wb['id'])
self.got_count += 1
# 这里是系统日志输出，尽量别太杂
logger.info('已获取用户 {} 的微博，内容为 {}'.format(self.user['screen_name'], wb['text']))
# self.print_weibo(wb)
else:
logger.info(u'正在过滤转发微博')
if const.CHECK_COOKIE['CHECK'] and not const.CHECK_COOKIE['CHECKED']:
logger.warning('经检查，cookie无效，系统退出')
sys.exit()
else:
return True
logger.info(u'{}已获取{}({})的第{}页微博{}'.format(
'-' * 30, self.user['screen_name'], self.user['id'], page,
'-' * 30))
except Exception as e:
logger.exception(e)
def get_page_count(self):
"""获取微博页数"""
try:
weibo_count = self.user['statuses_count']
page_count = int(math.ceil(weibo_count / 10.0))
return page_count
except KeyError:
logger.exception(
u'程序出错，错误原因可能为以下两者：\n'
u'1.user_id不正确；\n'
u'2.此用户微博可能需要设置cookie才能爬取。\n'
u'解决方案：\n'