cyberes
/
proxy-loadbalancer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 17 Wiki Activity

adjust logging and test headers

This commit is contained in:
Cyberes 2024-02-14 15:28:23 -07:00
parent 1cec50ddb3
commit 8c0e80efa8
3 changed files with 23 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
canihazip/server.py
View File

@ -11,4 +11,4 @@ def get_my_ip(path):
if __name__ == '__main__': if __name__ == '__main__':
app.run(host="0.0.0.0", port=7860) app.run(host="0.0.0.0", port=5000)

27
proxy-skeleton/app/background.py
View File

@ -13,6 +13,19 @@ from .pid import zombie_slayer
from .redis_cycle import add_backend_cycler from .redis_cycle import add_backend_cycler
from .smartproxy import transform_smartproxy from .smartproxy import transform_smartproxy
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'cross-site',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
}
def validate_proxies(): def validate_proxies():
""" """
@ -45,19 +58,19 @@ def validate_proxies():
smartproxy = False smartproxy = False
if pxy in SMARTPROXY_POOL: if pxy in SMARTPROXY_POOL:
smartproxy = True smartproxy = True
r = requests.get(IP_CHECKER, proxies={'http': transform_smartproxy(pxy), 'https': transform_smartproxy(pxy)}, timeout=15) r = requests.get(IP_CHECKER, proxies={'http': transform_smartproxy(pxy), 'https': transform_smartproxy(pxy)}, timeout=15, headers=headers)
# TODO: remove when fixed # TODO: remove when fixed
for d in SMARTPROXY_BV3HI_FIX: for d in SMARTPROXY_BV3HI_FIX:
r2 = requests.get(d, proxies={'http': transform_smartproxy(pxy), 'https': transform_smartproxy(pxy)}, timeout=15) r2 = requests.get(d, proxies={'http': transform_smartproxy(pxy), 'https': transform_smartproxy(pxy)}, timeout=15, headers=headers)
if r2.status_code != 200: if r2.status_code != 200:
logger.debug(f'PROXY BV3HI TEST failed - {pxy} - got code {r2.status_code}') logger.info(f'PROXY BV3HI TEST failed - {pxy} - got code {r2.status_code}')
return return
else: else:
r = requests.get(IP_CHECKER, proxies={'http': pxy, 'https': pxy}, timeout=15) r = requests.get(IP_CHECKER, proxies={'http': pxy, 'https': pxy}, timeout=15, headers=headers)
if r.status_code != 200: if r.status_code != 200:
logger.debug(f'PROXY TEST failed - {pxy} - got code {r.status_code}') logger.info(f'PROXY TEST failed - {pxy} - got code {r.status_code}')
return return
ip = r.text ip = r.text
@ -67,9 +80,9 @@ def validate_proxies():
proxy_dict[pxy] = ip proxy_dict[pxy] = ip
else: else:
s = ' Smartproxy ' if smartproxy else ' ' s = ' Smartproxy ' if smartproxy else ' '
logger.debug(f'Duplicate{s}IP: {ip}') logger.info(f'Duplicate{s}IP: {ip}')
except Exception as e: except Exception as e:
logger.debug(f'PROXY TEST failed - {pxy} - {e}') # ': {e.__class__.__name__}') logger.info(f'PROXY TEST failed - {pxy} - {e}') # ': {e.__class__.__name__}')
# traceback.print_exc() # traceback.print_exc()
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_PROXY_CHECKERS) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_PROXY_CHECKERS) as executor:

3
requirements.txt
View File

@ -3,4 +3,5 @@ redis
requests requests
coloredlogs coloredlogs
psutil psutil
flask flask
async_timeout