Compare commits
6 Commits
Author | SHA1 | Date |
---|---|---|
Cyberes | e93f353860 | |
Cyberes | c3e60eaca9 | |
Cyberes | 8a2ee75188 | |
Cyberes | a886056d7f | |
Cyberes | 2ce41d0637 | |
Cyberes | ea89052ef0 |
|
@ -6,6 +6,8 @@ This is a simple proxy load balancer that will route requests to a cluster of pr
|
|||
|
||||
This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported.
|
||||
|
||||
Memory usage sits at around 25M under load.
|
||||
|
||||
## Install
|
||||
|
||||
1. Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or run `./build.sh` to build the program locally.
|
||||
|
|
|
@ -4,6 +4,9 @@ http_port: 9000
|
|||
# How many proxies will be checked at once?
|
||||
proxy_checkers: 50
|
||||
|
||||
# The interval between proxy checks in seconds.
|
||||
proxy_check_interval: 60
|
||||
|
||||
# URL to get a proxy's IP.
|
||||
ip_checker_url: https://api.ipify.org
|
||||
|
||||
|
@ -32,3 +35,12 @@ thirdparty_bypass_domains:
|
|||
# Shuffle the proxy lists whenever the background thread refreshes them.
|
||||
# If false, round-robin on default order.
|
||||
shuffle_proxies: false
|
||||
|
||||
# Don't allow requests to these domains through the proxy.
|
||||
blocked_domains:
|
||||
- example.com
|
||||
|
||||
# Resolve specific domains through specific proxies.
|
||||
# Proxies here are not validated.
|
||||
resolve_through:
|
||||
github.com: http://1.2.3.4:3128
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
|
||||
redis
|
||||
requests
|
||||
coloredlogs
|
||||
psutil
|
||||
flask
|
||||
async_timeout
|
|
@ -19,6 +19,9 @@ type Config struct {
|
|||
ThirdpartyTestUrls []string
|
||||
ThirdpartyBypassDomains []string
|
||||
ShuffleProxies bool
|
||||
BlockedDomains []string
|
||||
ResolveThrough map[string]string
|
||||
ProxyCheckInterval int
|
||||
}
|
||||
|
||||
func SetConfig(configFile string) (*Config, error) {
|
||||
|
@ -36,6 +39,9 @@ func SetConfig(configFile string) (*Config, error) {
|
|||
viper.SetDefault("thirdparty_test_urls", make([]string, 0))
|
||||
viper.SetDefault("thirdparty_bypass_domains", make([]string, 0))
|
||||
viper.SetDefault("shuffle_proxies", false)
|
||||
viper.SetDefault("blocked_domains", make([]string, 0))
|
||||
viper.SetDefault("resolve_through", make(map[string]string))
|
||||
viper.SetDefault("proxy_check_interval", 60)
|
||||
|
||||
err := viper.ReadInConfig()
|
||||
if err != nil {
|
||||
|
@ -51,6 +57,9 @@ func SetConfig(configFile string) (*Config, error) {
|
|||
ThirdpartyTestUrls: viper.GetStringSlice("thirdparty_test_urls"),
|
||||
ThirdpartyBypassDomains: viper.GetStringSlice("thirdparty_bypass_domains"),
|
||||
ShuffleProxies: viper.GetBool("shuffle_proxies"),
|
||||
BlockedDomains: viper.GetStringSlice("blocked_domains"),
|
||||
ResolveThrough: viper.GetStringMapString("resolve_through"),
|
||||
ProxyCheckInterval: viper.GetInt("proxy_check_interval"),
|
||||
}
|
||||
|
||||
if len(config.ProxyPoolOurs) == 0 && len(config.ProxyPoolThirdparty) == 0 {
|
||||
|
|
|
@ -82,6 +82,8 @@ func main() {
|
|||
log.Fatalf(`Failed to load config: %s`, err)
|
||||
}
|
||||
|
||||
log.Debugf(`Proxy check interval: %d sec`, config.GetConfig().ProxyCheckInterval)
|
||||
|
||||
proxyCluster := proxy.NewForwardProxyCluster()
|
||||
go func() {
|
||||
log.Fatal(http.ListenAndServe(":"+configData.HTTPPort, proxyCluster))
|
||||
|
|
|
@ -25,17 +25,35 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
|
|||
Timeout: config.GetConfig().ProxyConnectTimeout,
|
||||
}
|
||||
|
||||
response, err := client.Get(targetURL)
|
||||
req, err := http.NewRequest("GET", targetURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode == http.StatusOK {
|
||||
bodyBytes, err := io.ReadAll(response.Body)
|
||||
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
req.Header.Set("Priority", "u=0, i")
|
||||
req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
|
||||
req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
|
||||
req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
|
||||
req.Header.Set("Sec-Fetch-Dest", "document")
|
||||
req.Header.Set("Sec-Fetch-Mode", "navigate")
|
||||
req.Header.Set("Sec-Fetch-Site", "cross-site")
|
||||
req.Header.Set("Sec-Fetch-User", "?1")
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bodyBytes), nil
|
||||
}
|
||||
return "", fmt.Errorf("bad response code %d", response.StatusCode)
|
||||
return "", fmt.Errorf("bad response code %d", resp.StatusCode)
|
||||
}
|
||||
|
|
|
@ -19,11 +19,17 @@ var (
|
|||
HeaderThirdpartyBypass = "Thirdparty-Bypass"
|
||||
)
|
||||
|
||||
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time) {
|
||||
log.Infof(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
|
||||
func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
|
||||
msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
|
||||
if *errorMsg == "" {
|
||||
log.Infof(msg)
|
||||
} else {
|
||||
log.Errorf(`%s -- %s`, msg, *errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
|
||||
urlHostname := req.URL.Hostname()
|
||||
if p.BalancerReady.GetCount() != 0 {
|
||||
errStr := "proxy is not ready"
|
||||
http.Error(w, errStr, http.StatusServiceUnavailable)
|
||||
|
@ -39,6 +45,12 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
|||
return "", "", "", "", nil, errors.New(errStr)
|
||||
}
|
||||
|
||||
if slices.Contains(config.GetConfig().BlockedDomains, urlHostname) {
|
||||
errStr := "this domain has been blocked"
|
||||
http.Error(w, errStr, http.StatusUnavailableForLegalReasons)
|
||||
return "", "", "", "", nil, errors.New(errStr)
|
||||
}
|
||||
|
||||
headerIncludeBrokenThirdparty := req.Header.Get(HeaderThirdpartyIncludeBroken)
|
||||
req.Header.Del(HeaderThirdpartyIncludeBroken)
|
||||
headerBypassThirdparty := req.Header.Get(HeaderThirdpartyBypass)
|
||||
|
@ -50,15 +62,20 @@ func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter,
|
|||
}
|
||||
|
||||
var selectedProxy string
|
||||
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, req.URL.Hostname()) {
|
||||
selectedProxy = p.getProxyFromOurs()
|
||||
|
||||
if val, ok := config.GetConfig().ResolveThrough[urlHostname]; ok {
|
||||
selectedProxy = val
|
||||
} else {
|
||||
if headerIncludeBrokenThirdparty != "" {
|
||||
selectedProxy = p.getProxyFromAllWithBroken()
|
||||
} else if headerBypassThirdparty != "" {
|
||||
if slices.Contains(config.GetConfig().ThirdpartyBypassDomains, urlHostname) {
|
||||
selectedProxy = p.getProxyFromOurs()
|
||||
} else {
|
||||
selectedProxy = p.getProxyFromAll()
|
||||
if headerIncludeBrokenThirdparty != "" {
|
||||
selectedProxy = p.getProxyFromAllWithBroken()
|
||||
} else if headerBypassThirdparty != "" {
|
||||
selectedProxy = p.getProxyFromOurs()
|
||||
} else {
|
||||
selectedProxy = p.getProxyFromAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
if selectedProxy == "" {
|
||||
|
@ -82,13 +99,21 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
_, proxyUser, proxyPass, proxyHost, parsedProxyUrl, err := p.validateRequestAndGetProxy(w, req)
|
||||
if err != nil {
|
||||
// Error has already been handled, just log and return.
|
||||
log.Debugf(`%s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, err)
|
||||
if proxyHost == "" {
|
||||
proxyHost = "none"
|
||||
}
|
||||
log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Variables for later
|
||||
var returnCode *int
|
||||
returnCode = new(int)
|
||||
*returnCode = -1
|
||||
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime)
|
||||
var errorMsg *string
|
||||
errorMsg = new(string)
|
||||
*errorMsg = ""
|
||||
defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
|
||||
|
||||
parsedProxyUrl.Scheme = "http"
|
||||
if proxyUser != "" && proxyPass != "" {
|
||||
|
@ -103,7 +128,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
|
||||
proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
@ -113,7 +138,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
|
|||
|
||||
resp, err := client.Do(proxyReq)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
|
||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
@ -132,18 +157,24 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
_, proxyUser, proxyPass, proxyHost, _, err := p.validateRequestAndGetProxy(w, req)
|
||||
if err != nil {
|
||||
// Error has already been handled, just log and return.
|
||||
log.Debugf(`%s -> %s -- CONNECT -- Rejecting request: %s`, remoteAddr, proxyHost, err)
|
||||
if proxyHost == "" {
|
||||
proxyHost = "none"
|
||||
}
|
||||
log.Debugf(`%s -> %s -> %s -- CONNECT -- Rejecting request: %s`, remoteAddr, proxyHost, targetHost, err)
|
||||
return
|
||||
}
|
||||
var returnCode *int
|
||||
returnCode = new(int)
|
||||
*returnCode = -1
|
||||
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime)
|
||||
var errorMsg *string
|
||||
errorMsg = new(string)
|
||||
*errorMsg = ""
|
||||
defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
|
||||
|
||||
// Start a connection to the downstream proxy server.
|
||||
proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to dial proxy %s - %s`, proxyHost, err)
|
||||
http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
@ -160,7 +191,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
}
|
||||
resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
|
||||
if resp == nil {
|
||||
log.Errorf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||
*errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
|
||||
http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
} else if err != nil {
|
||||
|
@ -171,14 +202,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
|
|||
w.WriteHeader(http.StatusOK)
|
||||
hj, ok := w.(http.Hijacker)
|
||||
if !ok {
|
||||
log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
||||
*errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
|
||||
http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
clientConn, _, err := hj.Hijack()
|
||||
if err != nil {
|
||||
log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
||||
*errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
|
||||
http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
|
|
@ -17,6 +17,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
ctx := context.TODO()
|
||||
|
||||
for {
|
||||
startTime := time.Now()
|
||||
p.refreshInProgress = true
|
||||
allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
|
||||
newOurOnlineProxies := make([]string, 0)
|
||||
|
@ -57,7 +58,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
// Test the proxy.
|
||||
ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
|
||||
if testErr != nil {
|
||||
log.Warnf("Validate - proxy %s failed: %s", proxyHost, testErr)
|
||||
log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
|
||||
if isThirdparty(pxy) {
|
||||
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||
} else {
|
||||
|
@ -66,7 +67,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
return
|
||||
}
|
||||
if slices.Contains(newIpAddresses, ipAddr) {
|
||||
log.Warnf("Validate - duplicate IP Address %s for proxy %s", ipAddr, proxyHost)
|
||||
log.Warnf("Validate - duplicate IP Address %s for %s", ipAddr, proxyHost)
|
||||
if isThirdparty(pxy) {
|
||||
newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
|
||||
} else {
|
||||
|
@ -78,15 +79,19 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
|
||||
// Sort the proxy into the right groups.
|
||||
if isThirdparty(pxy) {
|
||||
newThirdpartyOnlineProxies = append(newThirdpartyOnlineProxies, pxy)
|
||||
|
||||
okToAdd := true
|
||||
for _, d := range config.GetConfig().ThirdpartyTestUrls {
|
||||
_, bv3hiErr := sendRequestThroughProxy(pxy, d)
|
||||
if bv3hiErr != nil {
|
||||
log.Debugf("Validate - Third-party %s failed: %s", proxyHost, bv3hiErr)
|
||||
okToAdd = false
|
||||
newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
|
||||
log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr)
|
||||
break
|
||||
}
|
||||
}
|
||||
if okToAdd {
|
||||
newThirdpartyOnlineProxies = append(newThirdpartyOnlineProxies, pxy)
|
||||
}
|
||||
} else {
|
||||
newOurOnlineProxies = append(newOurOnlineProxies, pxy)
|
||||
}
|
||||
|
@ -117,12 +122,13 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
|
|||
}
|
||||
|
||||
p.mu.RLock()
|
||||
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d",
|
||||
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+(len(p.thirdpartyOnlineProxies)-len(p.thirdpartyBrokenProxies)))
|
||||
log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
|
||||
len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
|
||||
)
|
||||
p.mu.RUnlock()
|
||||
|
||||
p.refreshInProgress = false
|
||||
time.Sleep(60 * time.Second)
|
||||
time.Sleep(time.Duration(config.GetConfig().ProxyCheckInterval) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue