This commit is contained in:
MatMaul 2023-06-16 10:17:27 +00:00
parent fa7804d148
commit f0f2fe312c
2 changed files with 3 additions and 5 deletions

View File

@ -268,10 +268,8 @@ was enabled</em>, Catch-Up Mode is exited and we return to <code class="docutils
<h4 class="rubric" id="a-note-on-failures-and-back-offs">A note on failures and back-offs</h4>
<p>If a remote server is unreachable over federation, we back off from that server,
with an exponentially-increasing retry interval.
Whilst we dont automatically retry after the interval, we prevent making new attempts
until such time as the back-off has cleared.
Once the back-off is cleared and a new PDU or EDU arrives for transmission, the transmission
loop resumes and empties the queue by making federation requests.</p>
We automatically retry after the retry interval expires (roughly, the logic to do so
being triggered every minute).</p>
<p>If the backoff grows too large (&gt; 1 hour), the in-memory queue is emptied (to prevent
unbounded growth) and Catch-Up Mode is entered.</p>
<p>It is worth noting that the back-off for a remote server is cleared once an inbound

View File

@ -1 +1 @@
Search.setIndex({"docnames": ["index", "modules/federation_sender"], "filenames": ["index.rst", "modules/federation_sender.md"], "titles": ["Welcome to the Synapse Developer Documentation!", "Federation Sender"], "terms": {"feder": 0, "sender": 0, "index": 0, "modul": 0, "search": 0, "page": 0, "The": 1, "i": 1, "respons": 1, "send": 1, "persist": 1, "data": 1, "unit": 1, "pdu": 1, "ephemer": 1, "edu": 1, "other": 1, "homeserv": 1, "us": 1, "api": 1, "how": 1, "do": 1, "get": 1, "sent": 1, "made": 1, "awar": 1, "new": 1, "due": 1, "federationsend": 1, "notify_new_ev": 1, "when": 1, "notifi": 1, "about": 1, "newli": 1, "origin": 1, "from": 1, "thi": 1, "an": 1, "out": 1, "band": 1, "event": 1, "we": 1, "pass": 1, "_perdestinationqueu": 1, "each": 1, "remot": 1, "room": 1, "point": 1, "dag": 1, "per": 1, "destin": 1, "queue": 1, "There": 1, "one": 1, "perdestinationqueu": 1, "maintain": 1, "follow": 1, "inform": 1, "whether": 1, "current": 1, "catch": 1, "up": 1, "mode": 1, "see": 1, "below": 1, "consid": 1, "section": 1, "upon": 1, "being": 1, "enqueu": 1, "attempt_new_transact": 1, "call": 1, "start": 1, "transact": 1, "alreadi": 1, "progress": 1, "transmiss": 1, "loop": 1, "http": 1, "request": 1, "endpoint": 1, "refer": 1, "bodi": 1, "contain": 1, "list": 1, "_transaction_transmission_loop": 1, "empti": 1, "queu": 1, "them": 1, "can": 1, "onli": 1, "flight": 1, "given": 1, "ani": 1, "time": 1, "than": 1, "prevent": 1, "u": 1, "overload": 1, "also": 1, "make": 1, "easier": 1, "reason": 1, "becaus": 1, "process": 1, "sequenti": 1, "describ": 1, "later": 1, "continu": 1, "so": 1, "long": 1, "anyth": 1, "At": 1, "iter": 1, "dequeu": 1, "50": 1, "100": 1, "success": 1, "note": 1, "fact": 1, "succeed": 1, "transmit": 1, "stream_ord": 1, "latest": 1, "unsuccess": 1, "back": 1, "off": 1, "some": 1, "If": 1, "have": 1, "been": 1, "too": 1, "backoff": 1, "interv": 1, "grow": 1, "exce": 1, "1": 1, "hour": 1, "memori": 1, "ar": 1, "enter": 1, "ha": 1, "flag": 1, "set": 1, "_catch_up_transmission_loop": 1, "lieu": 1, "regular": 1, "onc": 1, "exit": 1, "tranact": 1, "behaviour": 1, "resum": 1, "synaps": 1, "startup": 1, "fallen": 1, "behind": 1, "connect": 1, "problem": 1, "miss": 1, "volatil": 1, "e": 1, "reset": 1, "doe": 1, "hold": 1, "forev": 1, "fail": 1, "mechan": 1, "last_successful_stream_ord": 1, "column": 1, "tabl": 1, "which": 1, "give": 1, "most": 1, "recent": 1, "successfulli": 1, "destination_room": 1, "need": 1, "pull": 1, "entri": 1, "oldest": 1, "": 1, "greater": 1, "In": 1, "word": 1, "These": 1, "could": 1, "principl": 1, "now": 1, "directli": 1, "howev": 1, "optimis": 1, "intend": 1, "instead": 1, "attempt": 1, "forward": 1, "extrem": 1, "still": 1, "elig": 1, "receiv": 1, "those": 1, "reduc": 1, "load": 1, "aggreg": 1, "all": 1, "behav": 1, "accord": 1, "therefor": 1, "avoid": 1, "lot": 1, "differ": 1, "recov": 1, "valid": 1, "partial": 1, "state": 1, "sinc": 1, "unabl": 1, "determin": 1, "unless": 1, "case": 1, "just": 1, "server": 1, "skip": 1, "whilst": 1, "through": 1, "posit": 1, "advanc": 1, "normal": 1, "longer": 1, "outstand": 1, "thei": 1, "arriv": 1, "wa": 1, "enabl": 1, "return": 1, "A": 1, "failur": 1, "unreach": 1, "over": 1, "exponenti": 1, "increas": 1, "retri": 1, "don": 1, "t": 1, "automat": 1, "after": 1, "until": 1, "clear": 1, "larg": 1, "unbound": 1, "growth": 1, "It": 1, "worth": 1, "inbound": 1, "notify_remote_server_up": 1, "proactiv": 1, "you": 1, "wait": 1, "creat": 1, "order": 1}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"welcom": 0, "synaps": 0, "develop": 0, "document": 0, "content": 0, "indic": 0, "tabl": 0, "feder": 1, "sender": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 57}, "alltitles": {"Welcome to the Synapse Developer Documentation!": [[0, "welcome-to-the-synapse-developer-documentation"]], "Contents:": [[0, null]], "Indices and tables": [[0, "indices-and-tables"]], "Federation Sender": [[1, "federation-sender"]]}, "indexentries": {}})
Search.setIndex({"docnames": ["index", "modules/federation_sender"], "filenames": ["index.rst", "modules/federation_sender.md"], "titles": ["Welcome to the Synapse Developer Documentation!", "Federation Sender"], "terms": {"feder": 0, "sender": 0, "index": 0, "modul": 0, "search": 0, "page": 0, "The": 1, "i": 1, "respons": 1, "send": 1, "persist": 1, "data": 1, "unit": 1, "pdu": 1, "ephemer": 1, "edu": 1, "other": 1, "homeserv": 1, "us": 1, "api": 1, "how": 1, "do": 1, "get": 1, "sent": 1, "made": 1, "awar": 1, "new": 1, "due": 1, "federationsend": 1, "notify_new_ev": 1, "when": 1, "notifi": 1, "about": 1, "newli": 1, "origin": 1, "from": 1, "thi": 1, "an": 1, "out": 1, "band": 1, "event": 1, "we": 1, "pass": 1, "_perdestinationqueu": 1, "each": 1, "remot": 1, "room": 1, "point": 1, "dag": 1, "per": 1, "destin": 1, "queue": 1, "There": 1, "one": 1, "perdestinationqueu": 1, "maintain": 1, "follow": 1, "inform": 1, "whether": 1, "current": 1, "catch": 1, "up": 1, "mode": 1, "see": 1, "below": 1, "consid": 1, "section": 1, "upon": 1, "being": 1, "enqueu": 1, "attempt_new_transact": 1, "call": 1, "start": 1, "transact": 1, "alreadi": 1, "progress": 1, "transmiss": 1, "loop": 1, "http": 1, "request": 1, "endpoint": 1, "refer": 1, "bodi": 1, "contain": 1, "list": 1, "_transaction_transmission_loop": 1, "empti": 1, "queu": 1, "them": 1, "can": 1, "onli": 1, "flight": 1, "given": 1, "ani": 1, "time": 1, "than": 1, "prevent": 1, "u": 1, "overload": 1, "also": 1, "make": 1, "easier": 1, "reason": 1, "becaus": 1, "process": 1, "sequenti": 1, "describ": 1, "later": 1, "continu": 1, "so": 1, "long": 1, "anyth": 1, "At": 1, "iter": 1, "dequeu": 1, "50": 1, "100": 1, "success": 1, "note": 1, "fact": 1, "succeed": 1, "transmit": 1, "stream_ord": 1, "latest": 1, "unsuccess": 1, "back": 1, "off": 1, "some": 1, "If": 1, "have": 1, "been": 1, "too": 1, "backoff": 1, "interv": 1, "grow": 1, "exce": 1, "1": 1, "hour": 1, "memori": 1, "ar": 1, "enter": 1, "ha": 1, "flag": 1, "set": 1, "_catch_up_transmission_loop": 1, "lieu": 1, "regular": 1, "onc": 1, "exit": 1, "tranact": 1, "behaviour": 1, "resum": 1, "synaps": 1, "startup": 1, "fallen": 1, "behind": 1, "connect": 1, "problem": 1, "miss": 1, "volatil": 1, "e": 1, "reset": 1, "doe": 1, "hold": 1, "forev": 1, "fail": 1, "mechan": 1, "last_successful_stream_ord": 1, "column": 1, "tabl": 1, "which": 1, "give": 1, "most": 1, "recent": 1, "successfulli": 1, "destination_room": 1, "need": 1, "pull": 1, "entri": 1, "oldest": 1, "": 1, "greater": 1, "In": 1, "word": 1, "These": 1, "could": 1, "principl": 1, "now": 1, "directli": 1, "howev": 1, "optimis": 1, "intend": 1, "instead": 1, "attempt": 1, "forward": 1, "extrem": 1, "still": 1, "elig": 1, "receiv": 1, "those": 1, "reduc": 1, "load": 1, "aggreg": 1, "all": 1, "behav": 1, "accord": 1, "therefor": 1, "avoid": 1, "lot": 1, "differ": 1, "recov": 1, "valid": 1, "partial": 1, "state": 1, "sinc": 1, "unabl": 1, "determin": 1, "unless": 1, "case": 1, "just": 1, "server": 1, "skip": 1, "whilst": 1, "through": 1, "posit": 1, "advanc": 1, "normal": 1, "longer": 1, "outstand": 1, "thei": 1, "arriv": 1, "wa": 1, "enabl": 1, "return": 1, "A": 1, "failur": 1, "unreach": 1, "over": 1, "exponenti": 1, "increas": 1, "retri": 1, "automat": 1, "after": 1, "expir": 1, "roughli": 1, "logic": 1, "trigger": 1, "everi": 1, "minut": 1, "larg": 1, "unbound": 1, "growth": 1, "It": 1, "worth": 1, "clear": 1, "inbound": 1, "notify_remote_server_up": 1, "proactiv": 1, "you": 1, "don": 1, "t": 1, "wait": 1, "creat": 1, "order": 1}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"welcom": 0, "synaps": 0, "develop": 0, "document": 0, "content": 0, "indic": 0, "tabl": 0, "feder": 1, "sender": 1}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx": 57}, "alltitles": {"Welcome to the Synapse Developer Documentation!": [[0, "welcome-to-the-synapse-developer-documentation"]], "Contents:": [[0, null]], "Indices and tables": [[0, "indices-and-tables"]], "Federation Sender": [[1, "federation-sender"]]}, "indexentries": {}})