Update app.py
Browse files
app.py
CHANGED
@@ -5,8 +5,7 @@ import asyncio
|
|
5 |
import datetime
|
6 |
import sys
|
7 |
import traceback
|
8 |
-
from aiohttp import web, ClientTimeout, TCPConnector
|
9 |
-
from urllib.parse import parse_qs
|
10 |
from collections import namedtuple
|
11 |
|
12 |
CacheEntry = namedtuple('CacheEntry', ['data', 'timestamp'])
|
@@ -30,19 +29,21 @@ cache = CustomCache(ttl=1800) # 30 minutes cache
|
|
30 |
|
31 |
CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
32 |
|
33 |
-
async def fetch_url(url, session, max_retries=
|
34 |
headers = {"User-Agent": CHROME_USER_AGENT}
|
35 |
for attempt in range(max_retries):
|
36 |
try:
|
37 |
-
|
|
|
38 |
response.raise_for_status()
|
39 |
content = await response.read()
|
40 |
return content.decode('utf-8', errors='ignore')
|
41 |
-
except
|
42 |
-
|
|
|
43 |
if attempt == max_retries - 1:
|
44 |
raise
|
45 |
-
await asyncio.sleep(
|
46 |
|
47 |
async def extract_and_transform_proxies(input_text):
|
48 |
try:
|
@@ -110,13 +111,11 @@ def get_client_ip(request):
|
|
110 |
|
111 |
async def handle_request(request):
|
112 |
if request.path == '/':
|
113 |
-
# 使用 request.query 替代 parse_qs
|
114 |
if 'url' in request.query:
|
115 |
url = request.query['url']
|
116 |
no_cache = 'nocache' in request.query
|
117 |
cache_entry = None if no_cache else cache.get(url)
|
118 |
cache_hit = False
|
119 |
-
new_data = False
|
120 |
|
121 |
if cache_entry and not no_cache:
|
122 |
result = cache_entry.data
|
@@ -131,23 +130,27 @@ async def handle_request(request):
|
|
131 |
if new_result != "未找到有效的代理配置" and new_result != "YAML解析错误":
|
132 |
result = new_result
|
133 |
cache.set(url, result)
|
134 |
-
new_data = True
|
135 |
cache_time = datetime.datetime.now()
|
136 |
elif not cache_hit:
|
137 |
result = new_result
|
138 |
cache_time = datetime.datetime.now()
|
139 |
except Exception as e:
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
144 |
|
145 |
proxy_count = result.count('\n') + 1 if result and result != "未找到有效的代理配置" else 0
|
146 |
return web.Response(text=result, content_type='text/plain', headers={
|
147 |
'X-Proxy-Count': str(proxy_count),
|
148 |
'X-Cache-Hit': str(cache_hit),
|
149 |
'X-Cache-Time': cache_time.strftime('%Y-%m-%d %H:%M:%S'),
|
150 |
-
'X-New-Data': str(new_data),
|
151 |
'X-No-Cache': str(no_cache)
|
152 |
})
|
153 |
else:
|
@@ -180,9 +183,8 @@ async def logging_middleware(request, handler):
|
|
180 |
proxy_count = response.headers.get('X-Proxy-Count', '0')
|
181 |
cache_hit = "Hit" if response.headers.get('X-Cache-Hit') == 'True' else "Miss"
|
182 |
cache_time = response.headers.get('X-Cache-Time', '-')
|
183 |
-
new_data = "Yes" if response.headers.get('X-New-Data') == 'True' else "No"
|
184 |
|
185 |
-
log_message = f"{timestamp} - {client_ip} - \"GET /?url={target_url}{'&nocache' if no_cache else ''}\" - Status: {status_code} - Proxies: {proxy_count} - Cache: {cache_hit} - CacheTime: {cache_time} -
|
186 |
print(log_message, flush=True)
|
187 |
|
188 |
return response
|
|
|
5 |
import datetime
|
6 |
import sys
|
7 |
import traceback
|
8 |
+
from aiohttp import web, ClientTimeout, TCPConnector, ClientError, ServerTimeoutError, TooManyRedirects
|
|
|
9 |
from collections import namedtuple
|
10 |
|
11 |
CacheEntry = namedtuple('CacheEntry', ['data', 'timestamp'])
|
|
|
29 |
|
30 |
CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
31 |
|
32 |
+
async def fetch_url(url, session, max_retries=5):
|
33 |
headers = {"User-Agent": CHROME_USER_AGENT}
|
34 |
for attempt in range(max_retries):
|
35 |
try:
|
36 |
+
timeout = ClientTimeout(total=40 * (attempt + 1)) # 递增的超时时间
|
37 |
+
async with session.get(url, headers=headers, timeout=timeout) as response:
|
38 |
response.raise_for_status()
|
39 |
content = await response.read()
|
40 |
return content.decode('utf-8', errors='ignore')
|
41 |
+
except (ClientError, asyncio.TimeoutError, ServerTimeoutError, TooManyRedirects) as e:
|
42 |
+
wait_time = 2 ** attempt # 指数退避
|
43 |
+
print(f"Attempt {attempt + 1} failed: {str(e)}. Retrying in {wait_time} seconds...", flush=True)
|
44 |
if attempt == max_retries - 1:
|
45 |
raise
|
46 |
+
await asyncio.sleep(wait_time)
|
47 |
|
48 |
async def extract_and_transform_proxies(input_text):
|
49 |
try:
|
|
|
111 |
|
112 |
async def handle_request(request):
|
113 |
if request.path == '/':
|
|
|
114 |
if 'url' in request.query:
|
115 |
url = request.query['url']
|
116 |
no_cache = 'nocache' in request.query
|
117 |
cache_entry = None if no_cache else cache.get(url)
|
118 |
cache_hit = False
|
|
|
119 |
|
120 |
if cache_entry and not no_cache:
|
121 |
result = cache_entry.data
|
|
|
130 |
if new_result != "未找到有效的代理配置" and new_result != "YAML解析错误":
|
131 |
result = new_result
|
132 |
cache.set(url, result)
|
|
|
133 |
cache_time = datetime.datetime.now()
|
134 |
elif not cache_hit:
|
135 |
result = new_result
|
136 |
cache_time = datetime.datetime.now()
|
137 |
except Exception as e:
|
138 |
+
error_message = f"Error processing request: {str(e)}"
|
139 |
+
print(error_message, flush=True)
|
140 |
+
traceback.print_exc()
|
141 |
+
if cache_entry:
|
142 |
+
print("Using cached data due to error", flush=True)
|
143 |
+
result = cache_entry.data
|
144 |
+
cache_time = cache_entry.timestamp
|
145 |
+
cache_hit = True
|
146 |
+
else:
|
147 |
+
return web.Response(text=error_message, status=500)
|
148 |
|
149 |
proxy_count = result.count('\n') + 1 if result and result != "未找到有效的代理配置" else 0
|
150 |
return web.Response(text=result, content_type='text/plain', headers={
|
151 |
'X-Proxy-Count': str(proxy_count),
|
152 |
'X-Cache-Hit': str(cache_hit),
|
153 |
'X-Cache-Time': cache_time.strftime('%Y-%m-%d %H:%M:%S'),
|
|
|
154 |
'X-No-Cache': str(no_cache)
|
155 |
})
|
156 |
else:
|
|
|
183 |
proxy_count = response.headers.get('X-Proxy-Count', '0')
|
184 |
cache_hit = "Hit" if response.headers.get('X-Cache-Hit') == 'True' else "Miss"
|
185 |
cache_time = response.headers.get('X-Cache-Time', '-')
|
|
|
186 |
|
187 |
+
log_message = f"{timestamp} - {client_ip} - \"GET /?url={target_url}{'&nocache' if no_cache else ''}\" - Status: {status_code} - Proxies: {proxy_count} - Cache: {cache_hit} - CacheTime: {cache_time} - NoCache: {'Yes' if no_cache else 'No'}"
|
188 |
print(log_message, flush=True)
|
189 |
|
190 |
return response
|