delete old file

README.md

@@ -6,6 +6,8 @@ This is a simple proxy load balancer that will route requests to a cluster of pr
 
 This proxy server will transparently forward HTTPS requests without terminating them, meaning a self-signed certificate is not required. Downstream HTTPS proxy servers are not supported.
 
+Memory usage sits at around 25M under load.
+
 ## Install
 
 1.  Download the latest release from [/releases](https://git.evulid.cc/cyberes/proxy-loadbalancer/releases) or run `./build.sh` to build the program locally.

requirements.txt

@@ -1,7 +0,0 @@
-proxy.py @ git+https://github.com/abhinavsingh/proxy.py.git@develop
-redis
-requests
-coloredlogs
-psutil
-flask
-async_timeout

src/proxy/checkProxy.go

@@ -25,17 +25,35 @@ func sendRequestThroughProxy(pxy string, targetURL string) (string, error) {
 		Timeout:   config.GetConfig().ProxyConnectTimeout,
 	}
 
-	response, err := client.Get(targetURL)
+	req, err := http.NewRequest("GET", targetURL, nil)
 	if err != nil {
 		return "", err
 	}
-	defer response.Body.Close()
-	if response.StatusCode == http.StatusOK {
-		bodyBytes, err := io.ReadAll(response.Body)
+
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.9")
+	req.Header.Set("Priority", "u=0, i")
+	req.Header.Set("Sec-Ch-Ua", `"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"`)
+	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
+	req.Header.Set("Sec-Ch-Ua-Platform", `"Windows"`)
+	req.Header.Set("Sec-Fetch-Dest", "document")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+	req.Header.Set("Sec-Fetch-Site", "cross-site")
+	req.Header.Set("Sec-Fetch-User", "?1")
+	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusOK {
+		bodyBytes, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return "", err
 		}
 		return string(bodyBytes), nil
 	}
-	return "", fmt.Errorf("bad response code %d", response.StatusCode)
+	return "", fmt.Errorf("bad response code %d", resp.StatusCode)
 }

src/proxy/handleConnect.go

@@ -19,8 +19,13 @@ var (
 	HeaderThirdpartyBypass        = "Thirdparty-Bypass"
 )
 
-func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time) {
-	log.Infof(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
+func logProxyRequest(remoteAddr string, proxyHost string, targetHost string, returnCode *int, proxyConnectMode string, requestStartTime time.Time, errorMsg *string) {
+	msg := fmt.Sprintf(`%s -> %s -> %s -> %d -- %s -- %d ms`, remoteAddr, proxyHost, targetHost, *returnCode, proxyConnectMode, time.Since(requestStartTime).Milliseconds())
+	if *errorMsg == "" {
+		log.Infof(msg)
+	} else {
+		log.Errorf(`%s -- %s`, msg, *errorMsg)
+	}
 }
 
 func (p *ForwardProxyCluster) validateRequestAndGetProxy(w http.ResponseWriter, req *http.Request) (string, string, string, string, *url.URL, error) {
@@ -100,10 +105,15 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 		log.Debugf(`%s -> %s -> %s -- HTTP -- Rejecting request: %s`, remoteAddr, proxyHost, req.Host, err)
 		return
 	}
+
+	// Variables for later
 	var returnCode *int
 	returnCode = new(int)
 	*returnCode = -1
-	defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime)
+	var errorMsg *string
+	errorMsg = new(string)
+	*errorMsg = ""
+	defer logProxyRequest(remoteAddr, proxyHost, req.Host, returnCode, "HTTP", requestStartTime, errorMsg)
 
 	parsedProxyUrl.Scheme = "http"
 	if proxyUser != "" && proxyPass != "" {
@@ -118,7 +128,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 
 	proxyReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
 	if err != nil {
-		log.Errorf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
+		*errorMsg = fmt.Sprintf(`Failed to make %s request to "%s": %s`, req.Method, req.URL.String(), err)
 		http.Error(w, "failed to make request to downstream", http.StatusInternalServerError)
 		return
 	}
@@ -128,7 +138,7 @@ func (p *ForwardProxyCluster) proxyHttpConnect(w http.ResponseWriter, req *http.
 
 	resp, err := client.Do(proxyReq)
 	if err != nil {
-		log.Errorf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
+		*errorMsg = fmt.Sprintf(`Failed to execute %s request to "%s": %s`, req.Method, req.URL.String(), err)
 		http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
 		return
 	}
@@ -156,12 +166,15 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	var returnCode *int
 	returnCode = new(int)
 	*returnCode = -1
-	defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime)
+	var errorMsg *string
+	errorMsg = new(string)
+	*errorMsg = ""
+	defer logProxyRequest(remoteAddr, proxyHost, targetHost, returnCode, "CONNECT", requestStartTime, errorMsg)
 
 	// Start a connection to the downstream proxy server.
 	proxyConn, err := net.DialTimeout("tcp", proxyHost, config.GetConfig().ProxyConnectTimeout)
 	if err != nil {
-		log.Errorf(`Failed to dial proxy %s - %s`, proxyHost, err)
+		*errorMsg = fmt.Sprintf(`Failed to dial proxy %s - %s`, proxyHost, err)
 		http.Error(w, "failed to make request to downstream", http.StatusServiceUnavailable)
 		return
 	}
@@ -178,7 +191,7 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	}
 	resp, err := http.ReadResponse(bufio.NewReader(proxyConn), req)
 	if resp == nil {
-		log.Errorf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
+		*errorMsg = fmt.Sprintf(`Failed to CONNECT to %s using proxy %s: %s`, req.Host, proxyHost, err)
 		http.Error(w, "failed to execute request to downstream", http.StatusServiceUnavailable)
 		return
 	} else if err != nil {
@@ -189,14 +202,14 @@ func (p *ForwardProxyCluster) proxyHttpsConnect(w http.ResponseWriter, req *http
 	w.WriteHeader(http.StatusOK)
 	hj, ok := w.(http.Hijacker)
 	if !ok {
-		log.Errorf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
+		*errorMsg = fmt.Sprintf(`Failed to forward connection to %s using proxy %s`, req.Host, proxyHost)
 		http.Error(w, "failed to forward connection to downstream", http.StatusServiceUnavailable)
 		return
 	}
 
 	clientConn, _, err := hj.Hijack()
 	if err != nil {
-		log.Errorf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
+		*errorMsg = fmt.Sprintf(`Failed to execute connection forwarding to %s using proxy %s`, req.Host, proxyHost)
 		http.Error(w, "failed to execute connection forwarding to downstream", http.StatusServiceUnavailable)
 		return
 	}

src/proxy/threads.go

@@ -17,6 +17,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 	ctx := context.TODO()
 
 	for {
+		startTime := time.Now()
 		p.refreshInProgress = true
 		allProxies := removeDuplicates(append(config.GetConfig().ProxyPoolOurs, config.GetConfig().ProxyPoolThirdparty...))
 		newOurOnlineProxies := make([]string, 0)
@@ -57,7 +58,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 				// Test the proxy.
 				ipAddr, testErr := sendRequestThroughProxy(pxy, config.GetConfig().IpCheckerURL)
 				if testErr != nil {
-					log.Warnf("Validate - proxy %s failed: %s", proxyHost, testErr)
+					log.Debugf("Validate - %s failed: %s", proxyHost, testErr)
 					if isThirdparty(pxy) {
 						newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
 					} else {
@@ -66,7 +67,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 					return
 				}
 				if slices.Contains(newIpAddresses, ipAddr) {
-					log.Warnf("Validate - duplicate IP Address %s for proxy %s", ipAddr, proxyHost)
+					log.Warnf("Validate - duplicate IP Address %s for %s", ipAddr, proxyHost)
 					if isThirdparty(pxy) {
 						newThirdpartyOfflineProxies = append(newThirdpartyOfflineProxies, pxy)
 					} else {
@@ -84,7 +85,7 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 						if bv3hiErr != nil {
 							okToAdd = false
 							newThirdpartyBrokenProxies = append(newThirdpartyBrokenProxies, pxy)
-							log.Debugf("Validate - Third-party %s failed: %s", proxyHost, bv3hiErr)
+							log.Debugf("Validate - %s failed third-party test: %s", proxyHost, bv3hiErr)
 							break
 						}
 					}
@@ -121,8 +122,8 @@ func (p *ForwardProxyCluster) ValidateProxiesThread() {
 		}
 
 		p.mu.RLock()
-		log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d",
-			len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies),
+		log.Infof("Our Endpoints Online: %d, Third-Party Endpoints Online: %d, Third-Party Broken Endpoints: %d, Total Valid: %d, Elapsed: %s",
+			len(p.ourOnlineProxies), len(p.thirdpartyOnlineProxies), len(p.thirdpartyBrokenProxies), len(p.ourOnlineProxies)+len(p.thirdpartyOnlineProxies), time.Since(startTime),
 		)
 		p.mu.RUnlock()