Only allow webhook to send requests to allowed hosts (#17482) (#17510)

Backport #17482

* Only allow webhook to send requests to allowed hosts (backport #17482)

* use ALLOWED_HOST_LIST=* for default to keep the legacy behavior in 1.15.x

cmd/web.go

@@ -194,6 +194,10 @@ func listen(m http.Handler, handleRedirector bool) error {
 		listenAddr = net.JoinHostPort(listenAddr, setting.HTTPPort)
 	}
 	log.Info("Listen: %v://%s%s", setting.Protocol, listenAddr, setting.AppSubURL)
+	// This can be useful for users, many users do wrong to their config and get strange behaviors behind a reverse-proxy.
+	// A user may fix the configuration mistake when he sees this log.
+	// And this is also very helpful to maintainers to provide help to users to resolve their configuration problems.
+	log.Info("AppURL(ROOT_URL): %s", setting.AppURL)
 
 	if setting.LFS.StartServer {
 		log.Info("LFS server enabled")

custom/conf/app.example.ini

@@ -1388,6 +1388,13 @@ PATH =
 ;; Deliver timeout in seconds
 ;DELIVER_TIMEOUT = 5
 ;;
+;; Webhook can only call allowed hosts for security reasons. Comma separated list, eg: external, 192.168.1.0/24, *.mydomain.com
+;; Built-in: loopback (for localhost), private (for LAN/intranet), external (for public hosts on internet), * (for all hosts)
+;; CIDR list: 1.2.3.0/8, 2001:db8::/32
+;; Wildcard hosts: *.mydomain.com, 192.168.100.*
+;; Default to * for 1.15.x, external for 1.16 and later
+;ALLOWED_HOST_LIST = *
+;;
 ;; Allow insecure certification
 ;SKIP_TLS_VERIFY = false
 ;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -545,6 +545,14 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
 
 - `QUEUE_LENGTH`: **1000**: Hook task queue length. Use caution when editing this value.
 - `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
+- `ALLOWED_HOST_LIST`: `*`: Default to `*` for 1.15.x, `external` for 1.16 and later. Webhook can only call allowed hosts for security reasons. Comma separated list.
+  - Built-in networks:
+    - `loopback`: 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+    - `private`: RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+    - `external`: A valid non-private unicast IP, you can access all hosts on public internet. 
+    - `*`: All hosts are allowed.
+  - CIDR list: `1.2.3.0/8` for IPv4 and `2001:db8::/32` for IPv6
+  - Wildcard hosts: `*.mydomain.com`, `192.168.100.*`
 - `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
 - `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
 - `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy

modules/hostmatcher/hostmatcher.go

@@ -0,0 +1,94 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package hostmatcher
+
+import (
+	"net"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// HostMatchList is used to check if a host or IP is in a list.
+// If you only need to do wildcard matching, consider to use modules/matchlist
+type HostMatchList struct {
+	hosts  []string
+	ipNets []*net.IPNet
+}
+
+// MatchBuiltinAll all hosts are matched
+const MatchBuiltinAll = "*"
+
+// MatchBuiltinExternal A valid non-private unicast IP, all hosts on public internet are matched
+const MatchBuiltinExternal = "external"
+
+// MatchBuiltinPrivate RFC 1918 (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16) and RFC 4193 (FC00::/7). Also called LAN/Intranet.
+const MatchBuiltinPrivate = "private"
+
+// MatchBuiltinLoopback 127.0.0.0/8 for IPv4 and ::1/128 for IPv6, localhost is included.
+const MatchBuiltinLoopback = "loopback"
+
+// ParseHostMatchList parses the host list HostMatchList
+func ParseHostMatchList(hostList string) *HostMatchList {
+	hl := &HostMatchList{}
+	for _, s := range strings.Split(hostList, ",") {
+		s = strings.ToLower(strings.TrimSpace(s))
+		if s == "" {
+			continue
+		}
+		_, ipNet, err := net.ParseCIDR(s)
+		if err == nil {
+			hl.ipNets = append(hl.ipNets, ipNet)
+		} else {
+			hl.hosts = append(hl.hosts, s)
+		}
+	}
+	return hl
+}
+
+// MatchesHostOrIP checks if the host or IP matches an allow/deny(block) list
+func (hl *HostMatchList) MatchesHostOrIP(host string, ip net.IP) bool {
+	var matched bool
+	host = strings.ToLower(host)
+	ipStr := ip.String()
+loop:
+	for _, hostInList := range hl.hosts {
+		switch hostInList {
+		case "":
+			continue
+		case MatchBuiltinAll:
+			matched = true
+			break loop
+		case MatchBuiltinExternal:
+			if matched = ip.IsGlobalUnicast() && !util.IsIPPrivate(ip); matched {
+				break loop
+			}
+		case MatchBuiltinPrivate:
+			if matched = util.IsIPPrivate(ip); matched {
+				break loop
+			}
+		case MatchBuiltinLoopback:
+			if matched = ip.IsLoopback(); matched {
+				break loop
+			}
+		default:
+			if matched, _ = filepath.Match(hostInList, host); matched {
+				break loop
+			}
+			if matched, _ = filepath.Match(hostInList, ipStr); matched {
+				break loop
+			}
+		}
+	}
+	if !matched {
+		for _, ipNet := range hl.ipNets {
+			if matched = ipNet.Contains(ip); matched {
+				break
+			}
+		}
+	}
+	return matched
+}

modules/hostmatcher/hostmatcher_test.go

@@ -0,0 +1,119 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package hostmatcher
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHostOrIPMatchesList(t *testing.T) {
+	type tc struct {
+		host     string
+		ip       net.IP
+		expected bool
+	}
+
+	// for IPv6: "::1" is loopback, "fd00::/8" is private
+
+	hl := ParseHostMatchList("private, External, *.myDomain.com, 169.254.1.0/24")
+	cases := []tc{
+		{"", net.IPv4zero, false},
+		{"", net.IPv6zero, false},
+
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("::1"), false},
+
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("fd00::1"), true},
+
+		{"", net.ParseIP("8.8.8.8"), true},
+		{"", net.ParseIP("1001::1"), true},
+
+		{"mydomain.com", net.IPv4zero, false},
+		{"sub.mydomain.com", net.IPv4zero, true},
+
+		{"", net.ParseIP("169.254.1.1"), true},
+		{"", net.ParseIP("169.254.2.2"), false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("loopback")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), true},
+		{"", net.ParseIP("10.0.1.1"), false},
+		{"", net.ParseIP("192.168.1.1"), false},
+		{"", net.ParseIP("8.8.8.8"), false},
+
+		{"", net.ParseIP("::1"), true},
+		{"", net.ParseIP("fd00::1"), false},
+		{"", net.ParseIP("1000::1"), false},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("private")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("8.8.8.8"), false},
+
+		{"", net.ParseIP("::1"), false},
+		{"", net.ParseIP("fd00::1"), true},
+		{"", net.ParseIP("1000::1"), false},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("external")
+	cases = []tc{
+		{"", net.IPv4zero, false},
+		{"", net.ParseIP("127.0.0.1"), false},
+		{"", net.ParseIP("10.0.1.1"), false},
+		{"", net.ParseIP("192.168.1.1"), false},
+		{"", net.ParseIP("8.8.8.8"), true},
+
+		{"", net.ParseIP("::1"), false},
+		{"", net.ParseIP("fd00::1"), false},
+		{"", net.ParseIP("1000::1"), true},
+
+		{"mydomain.com", net.IPv4zero, false},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+
+	hl = ParseHostMatchList("*")
+	cases = []tc{
+		{"", net.IPv4zero, true},
+		{"", net.ParseIP("127.0.0.1"), true},
+		{"", net.ParseIP("10.0.1.1"), true},
+		{"", net.ParseIP("192.168.1.1"), true},
+		{"", net.ParseIP("8.8.8.8"), true},
+
+		{"", net.ParseIP("::1"), true},
+		{"", net.ParseIP("fd00::1"), true},
+		{"", net.ParseIP("1000::1"), true},
+
+		{"mydomain.com", net.IPv4zero, true},
+	}
+	for _, c := range cases {
+		assert.Equalf(t, c.expected, hl.MatchesHostOrIP(c.host, c.ip), "case %s(%v)", c.host, c.ip)
+	}
+}

modules/migrations/migrate.go

@@ -89,7 +89,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
 			return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
 		}
 		for _, addr := range addrList {
-			if isIPPrivate(addr) || !addr.IsGlobalUnicast() {
+			if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
 				return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
 			}
 		}
@@ -486,16 +486,3 @@ func Init() error {
 
 	return nil
 }
-
-// isIPPrivate reports whether ip is a private address, according to
-// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses).
-// from https://github.com/golang/go/pull/42793
-// TODO remove if https://github.com/golang/go/issues/29146 got resolved
-func isIPPrivate(ip net.IP) bool {
-	if ip4 := ip.To4(); ip4 != nil {
-		return ip4[0] == 10 ||
-			(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
-			(ip4[0] == 192 && ip4[1] == 168)
-	}
-	return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
-}

modules/setting/webhook.go

@@ -7,20 +7,22 @@ package setting
 import (
 	"net/url"
 
+	"code.gitea.io/gitea/modules/hostmatcher"
 	"code.gitea.io/gitea/modules/log"
 )
 
 var (
 	// Webhook settings
 	Webhook = struct {
-		QueueLength    int
+		QueueLength     int
-		DeliverTimeout int
+		DeliverTimeout  int
-		SkipTLSVerify  bool
+		SkipTLSVerify   bool
-		Types          []string
+		AllowedHostList *hostmatcher.HostMatchList
-		PagingNum      int
+		Types           []string
-		ProxyURL       string
+		PagingNum       int
-		ProxyURLFixed  *url.URL
+		ProxyURL        string
-		ProxyHosts     []string
+		ProxyURLFixed   *url.URL
+		ProxyHosts      []string
 	}{
 		QueueLength:    1000,
 		DeliverTimeout: 5,
@@ -36,6 +38,7 @@ func newWebhookService() {
 	Webhook.QueueLength = sec.Key("QUEUE_LENGTH").MustInt(1000)
 	Webhook.DeliverTimeout = sec.Key("DELIVER_TIMEOUT").MustInt(5)
 	Webhook.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool()
+	Webhook.AllowedHostList = hostmatcher.ParseHostMatchList(sec.Key("ALLOWED_HOST_LIST").MustString(hostmatcher.MatchBuiltinAll))
 	Webhook.Types = []string{"gitea", "gogs", "slack", "discord", "dingtalk", "telegram", "msteams", "feishu", "matrix"}
 	Webhook.PagingNum = sec.Key("PAGING_NUM").MustInt(10)
 	Webhook.ProxyURL = sec.Key("PROXY_URL").MustString("")

modules/util/net.go

@@ -0,0 +1,19 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package util
+
+import (
+	"net"
+)
+
+// IsIPPrivate for net.IP.IsPrivate. TODO: replace with `ip.IsPrivate()` if min go version is bumped to 1.17
+func IsIPPrivate(ip net.IP) bool {
+	if ip4 := ip.To4(); ip4 != nil {
+		return ip4[0] == 10 ||
+			(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
+			(ip4[0] == 192 && ip4[1] == 168)
+	}
+	return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
+}

services/webhook/deliver.go

@@ -20,6 +20,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"code.gitea.io/gitea/models"
@@ -29,6 +30,8 @@ import (
 	"github.com/gobwas/glob"
 )
 
+var contextKeyWebhookRequest interface{} = "contextKeyWebhookRequest"
+
 // Deliver deliver hook task
 func Deliver(t *models.HookTask) error {
 	w, err := models.GetWebhookByID(t.HookID)
@@ -166,7 +169,7 @@ func Deliver(t *models.HookTask) error {
 		return fmt.Errorf("Webhook task skipped (webhooks disabled): [%d]", t.ID)
 	}
 
-	resp, err := webhookHTTPClient.Do(req)
+	resp, err := webhookHTTPClient.Do(req.WithContext(context.WithValue(req.Context(), contextKeyWebhookRequest, req)))
 	if err != nil {
 		t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
 		return err
@@ -288,14 +291,29 @@ func InitDeliverHooks() {
 	timeout := time.Duration(setting.Webhook.DeliverTimeout) * time.Second
 
 	webhookHTTPClient = &http.Client{
+		Timeout: timeout,
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Webhook.SkipTLSVerify},
 			Proxy:           webhookProxy(),
-			Dial: func(netw, addr string) (net.Conn, error) {
+			DialContext: func(ctx context.Context, network, addrOrHost string) (net.Conn, error) {
-				return net.DialTimeout(netw, addr, timeout) // dial timeout
+				dialer := net.Dialer{
+					Timeout: timeout,
+					Control: func(network, ipAddr string, c syscall.RawConn) error {
+						// in Control func, the addr was already resolved to IP:PORT format, there is no cost to do ResolveTCPAddr here
+						tcpAddr, err := net.ResolveTCPAddr(network, ipAddr)
+						req := ctx.Value(contextKeyWebhookRequest).(*http.Request)
+						if err != nil {
+							return fmt.Errorf("webhook can only call HTTP servers via TCP, deny '%s(%s:%s)', err=%v", req.Host, network, ipAddr, err)
+						}
+						if !setting.Webhook.AllowedHostList.MatchesHostOrIP(req.Host, tcpAddr.IP) {
+							return fmt.Errorf("webhook can only call allowed HTTP servers (check your webhook.ALLOWED_HOST_LIST setting), deny '%s(%s)'", req.Host, ipAddr)
+						}
+						return nil
+					},
+				}
+				return dialer.DialContext(ctx, network, addrOrHost)
 			},
 		},
-		Timeout: timeout, // request timeout
 	}
 
 	go graceful.GetManager().RunWithShutdownContext(DeliverHooks)