-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmain.py
203 lines (167 loc) · 5.85 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
import time
import requests
import pandas
import sys
import os
from bs4 import BeautifulSoup
import json
import datetime
from dotenv import load_dotenv
load_dotenv()
# # 目標網址
# target_url = "https://www.ptt.cc/bbs/"
# # 目標看板
# target_board = "Tech_Job"
# #目標頁面
# target_page = "/index"
# #目標頁數
# page_num = ""
# #頁面附屬檔名
# page_ext = ".html"
# target = target_url + target_board + target_page + page_num + page_ext
# headers = {
# "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
# }
# 常數
# 目標網址
TARGET_URL = "https://www.ptt.cc/bbs/"
# 目標頁面
TARGET_PAGE = "/index"
# 頁面附屬檔名
PAGE_EXT = ".html"
# Requests 請求標頭
HEADERS = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64;x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
}
def download_html(target, headers):
# deta = requests.get(target, headers= headers)
# return data這兩行如同下面
return requests.get(target, headers= headers)
def parser_board_urls():
return None
def parser_article_content():
return None
## Test
# a = download_html(target, headers)
# print(a.content)
def parser_board_urls(requests_):
# 傳入 request_ 並解析成 bs4 物件
html_code = BeautifulSoup(requests_.content, features="html.parser")
div_list = html_code.find_all('div', class_='title')
urls = []
for div_ in div_list:
try:
a_tag = div_.find('a').attrs['href']
except:
a_tag = None
urls.append(a_tag)
return urls
#可用以下程式碼進行測試
# a = download_html(target, headers)
# b = parser_board_urls(a)
# for u in b:
# print(u)
def parser_article_content(url_list):
ptt_data = []
for url_ in url_list:
if url_ != None:
article_url = 'https://www.ptt.cc' + url_
page_data = download_html(article_url, HEADERS)
page_html_code = BeautifulSoup(page_data.content, features="html.parser")
try:
article_data = page_html_code.find_all('span', class_="article-meta-value")
article_author = article_data[0].contents[0]
article_title = article_data[2].contents[0]
article_time = article_data[3].contents[0]
article_body = page_html_code.find('div', id='main-content').contents[4]
article_row = {
'url': article_url,
'title': article_title,
'author': article_author,
'time': article_time,
'content': article_body
}
ptt_data.append(article_row)
except:
print("parser error:", article_url)
time.sleep(1)
return ptt_data
# function Test
# a = download_html(target, headers)
# b = parser_board_urls(a)
# c = parser_article_content(b)
# print(c)
def export_json(data):
ptt_df = pandas.DataFrame(data)
ptt_df.to_json("ptt.json")
return True
# # Test
# a = download_html(target, headers)
# b = parser_board_urls(a)
# c = parser_article_content(b)
# export_json(c)
# print(c)
def main():
print("缺少參數長度",len(sys.argv))
if len(sys.argv)<2:
print("缺少參數:爬蟲的目標刊版")
sys.exit()
else:
if len(sys.argv) == 2:
page_num = " "
print("未偵測到目標頁數，因此只進行最新文章頁面進行爬蟲")
else:
#.....主程式碼...
# print("接收參數的長度:", len(sys.argv))
# print("參數內容 (list):", sys.argv)
# print("印出第一個參數:", sys.argv[0])
# print("印出第二個參數:", sys.argv[1])
#print("印出第三個參數:", sys.argv[2])
page_num = sys.argv[2]
target_board = sys.argv[1]
target = TARGET_URL + target_board +TARGET_PAGE + page_num + PAGE_EXT
print(target)
# '''爬蟲執行順序
# download_html() -> parser_board_urls() -> parser_article_content() -> export_json()
# download_html: arg1: 網址, arg2: 請求標頭, returns: requests class
# parser_board_urls: arg1: request class, returns: (list) urls
# parser_article_content: arg1: (list) urls, return: (list) ppt_data
# export_json: arg1: (list) ppt_data, return: (Boolean) True, Output: ppt.json
# '''
# 爬蟲主程式
res = download_html(target, HEADERS)
url_list = parser_board_urls(res)
ptt_data = parser_article_content(url_list)
export_json(ptt_data)
print(ptt_data)
# 發動訊息
time_str = "{date:%Y-%m-%d %H:%M:%S}".format(date=datetime.datetime.now())
process_st = send_line_notify("{} - 看板{} 爬蟲已完成".format(time_str, target_board))
if process_st == 200:
print("Line Notify 通知完成。")
else:
print("Line Notify API Error")
return None
def send_line_notify(msg = '傳輸訊息'):
line_notify_url = "https://notify-api.line.me/api/notify"
line_notify_token = os.getenv("Token")
# Line Auth Header
line_notify_header = {
'Authorization': 'Bearer {}'.format(line_notify_token)
}
# Line Message
line_notify_body = {
'message': msg
}
res = requests.post(line_notify_url, headers=line_notify_header, data = line_notify_body)
res_msg = json.loads(res.text)
return res_msg['status']
# print(send_line_notify("123456789"))
# print(os.getenv("Test"))
# from dotenv import dotenv_values
# config = dotenv_values(".env")
# print(config['Test'])
# 加入 __main__ 執行區段
if __name__ == '__main__':
main()
#main()