2021-08-10 07:24:16 -06:00
<!DOCTYPE HTML>
< html lang = "en" class = "sidebar-visible no-js light" >
< head >
<!-- Book generated using mdBook -->
< meta charset = "UTF-8" >
< title > Room DAG concepts - Synapse< / title >
<!-- Custom HTML head -->
< meta content = "text/html; charset=utf-8" http-equiv = "Content-Type" >
< meta name = "description" content = "" >
< meta name = "viewport" content = "width=device-width, initial-scale=1" >
< meta name = "theme-color" content = "#ffffff" / >
< link rel = "icon" href = "../favicon.svg" >
< link rel = "shortcut icon" href = "../favicon.png" >
< link rel = "stylesheet" href = "../css/variables.css" >
< link rel = "stylesheet" href = "../css/general.css" >
< link rel = "stylesheet" href = "../css/chrome.css" >
< link rel = "stylesheet" href = "../css/print.css" media = "print" >
<!-- Fonts -->
< link rel = "stylesheet" href = "../FontAwesome/css/font-awesome.css" >
< link rel = "stylesheet" href = "../fonts/fonts.css" >
<!-- Highlight.js Stylesheets -->
< link rel = "stylesheet" href = "../highlight.css" >
< link rel = "stylesheet" href = "../tomorrow-night.css" >
< link rel = "stylesheet" href = "../ayu-highlight.css" >
<!-- Custom theme stylesheets -->
< link rel = "stylesheet" href = "../docs/website_files/table-of-contents.css" >
< link rel = "stylesheet" href = "../docs/website_files/remove-nav-buttons.css" >
< link rel = "stylesheet" href = "../docs/website_files/indent-section-headers.css" >
< / head >
< body >
<!-- Provide site root to javascript -->
< script type = "text/javascript" >
var path_to_root = "../";
var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
< / script >
<!-- Work around some values being stored in localStorage wrapped in quotes -->
< script type = "text/javascript" >
try {
var theme = localStorage.getItem('mdbook-theme');
var sidebar = localStorage.getItem('mdbook-sidebar');
if (theme.startsWith('"') & & theme.endsWith('"')) {
localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
}
if (sidebar.startsWith('"') & & sidebar.endsWith('"')) {
localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
}
} catch (e) { }
< / script >
<!-- Set the theme before any content is loaded, prevents flash -->
< script type = "text/javascript" >
var theme;
try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
if (theme === null || theme === undefined) { theme = default_theme; }
var html = document.querySelector('html');
html.classList.remove('no-js')
html.classList.remove('light')
html.classList.add(theme);
html.classList.add('js');
< / script >
<!-- Hide / unhide sidebar before it is displayed -->
< script type = "text/javascript" >
var html = document.querySelector('html');
var sidebar = 'hidden';
if (document.body.clientWidth >= 1080) {
try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
sidebar = sidebar || 'visible';
}
html.classList.remove('sidebar-visible');
html.classList.add("sidebar-" + sidebar);
< / script >
< nav id = "sidebar" class = "sidebar" aria-label = "Table of contents" >
< div class = "sidebar-scrollbox" >
2022-11-22 05:22:40 -07:00
< ol class = "chapter" > < li class = "chapter-item expanded affix " > < li class = "part-title" > Introduction< / li > < li class = "chapter-item expanded " > < a href = "../welcome_and_overview.html" > Welcome and Overview< / a > < / li > < li class = "chapter-item expanded affix " > < li class = "part-title" > Setup< / li > < li class = "chapter-item expanded " > < a href = "../setup/installation.html" > Installation< / a > < / li > < li class = "chapter-item expanded " > < a href = "../postgres.html" > Using Postgres< / a > < / li > < li class = "chapter-item expanded " > < a href = "../reverse_proxy.html" > Configuring a Reverse Proxy< / a > < / li > < li class = "chapter-item expanded " > < a href = "../setup/forward_proxy.html" > Configuring a Forward/Outbound Proxy< / a > < / li > < li class = "chapter-item expanded " > < a href = "../turn-howto.html" > Configuring a Turn Server< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded " > < a href = "../setup/turn/coturn.html" > coturn TURN server< / a > < / li > < li class = "chapter-item expanded " > < a href = "../setup/turn/eturnal.html" > eturnal TURN server< / a > < / li > < / ol > < / li > < li class = "chapter-item expanded " > < a href = "../delegate.html" > Delegation< / a > < / li > < li class = "chapter-item expanded affix " > < li class = "part-title" > Upgrading< / li > < li class = "chapter-item expanded " > < a href = "../upgrade.html" > Upgrading between Synapse Versions< / a > < / li > < li class = "chapter-item expanded affix " > < li class = "part-title" > Usage< / li > < li class = "chapter-item expanded " > < a href = "../federate.html" > Federation< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/index.html" > Configuration< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded " > < a href = "../usage/configuration/config_documentation.html" > Configuration Manual< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/homeserver_sample_config.html" > Homeserver Sample Config File< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/logging_sample_config.html" > Logging Sample Config File< / a > < / li > < li class = "chapter-item expanded " > < a href = "../structured_logging.html" > Structured Logging< / a > < / li > < li class = "chapter-item expanded " > < a href = "../templates.html" > Templates< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/user_authentication/index.html" > User Authentication< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded " > < a href = "../usage/configuration/user_authentication/single_sign_on/index.html" > Single-Sign On< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded " > < a href = "../openid.html" > OpenID Connect< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/user_authentication/single_sign_on/saml.html" > SAML< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/user_authentication/single_sign_on/cas.html" > CAS< / a > < / li > < li class = "chapter-item expanded " > < a href = "../sso_mapping_providers.html" > SSO Mapping Providers< / a > < / li > < / ol > < / li > < li class = "chapter-item expanded " > < a href = "../password_auth_providers.html" > Password Auth Providers< / a > < / li > < li class = "chapter-item expanded " > < a href = "../jwt.html" > JSON Web Tokens< / a > < / li > < li class = "chapter-item expanded " > < a href = "../usage/configuration/user_authentication/refresh_tokens.html" > Refresh Tokens< / a > < / li > < / ol > < / li > < li class = "chapter-item expanded " > < a href = "../CAPTCHA_SETUP.html" > Registration Captcha< / a > < / li > < li class = "chapter-item expanded " > < a href = "../application_services.html" > Application Services< / a > < / li > < li class = "chapter-item expanded " > < a href = "../server_notices.html" > Server Notices< / a > < / li > < li class = "chapter-item expanded " > < a href = "../consent_tracking.html" > Consent Tracking< / a > < / li > < li class = "chapter-item expanded " > < a href = "../user_directory.html" > User Directory< / a > < / li > < li class = "chapter-item expanded " > < a href = "../message_retention_policies.html" > Message Retention Policies< / a > < / li > < li class = "chapter-item expanded " > < a href = "../modules/index.html" > Pluggable Modules< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded " > < a href = "../modules/writing_a_module.html" > Writing a module< / a > < / li > < li > < ol class = "section" > < li class = "chapter-item expanded "
2021-08-10 07:24:16 -06:00
< / div >
< div id = "sidebar-resize-handle" class = "sidebar-resize-handle" > < / div >
< / nav >
< div id = "page-wrapper" class = "page-wrapper" >
< div class = "page" >
< div id = "menu-bar-hover-placeholder" > < / div >
< div id = "menu-bar" class = "menu-bar sticky bordered" >
< div class = "left-buttons" >
< button id = "sidebar-toggle" class = "icon-button" type = "button" title = "Toggle Table of Contents" aria-label = "Toggle Table of Contents" aria-controls = "sidebar" >
< i class = "fa fa-bars" > < / i >
< / button >
< button id = "theme-toggle" class = "icon-button" type = "button" title = "Change theme" aria-label = "Change theme" aria-haspopup = "true" aria-expanded = "false" aria-controls = "theme-list" >
< i class = "fa fa-paint-brush" > < / i >
< / button >
< ul id = "theme-list" class = "theme-popup" aria-label = "Themes" role = "menu" >
< li role = "none" > < button role = "menuitem" class = "theme" id = "light" > Light (default)< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "rust" > Rust< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "coal" > Coal< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "navy" > Navy< / button > < / li >
< li role = "none" > < button role = "menuitem" class = "theme" id = "ayu" > Ayu< / button > < / li >
< / ul >
< button id = "search-toggle" class = "icon-button" type = "button" title = "Search. (Shortkey: s)" aria-label = "Toggle Searchbar" aria-expanded = "false" aria-keyshortcuts = "S" aria-controls = "searchbar" >
< i class = "fa fa-search" > < / i >
< / button >
< / div >
< h1 class = "menu-title" > Synapse< / h1 >
< div class = "right-buttons" >
< a href = "../print.html" title = "Print this book" aria-label = "Print this book" >
< i id = "print-button" class = "fa fa-print" > < / i >
< / a >
< a href = "https://github.com/matrix-org/synapse" title = "Git repository" aria-label = "Git repository" >
< i id = "git-repository-button" class = "fa fa-github" > < / i >
< / a >
< a href = "https://github.com/matrix-org/synapse/edit/develop/docs/development/room-dag-concepts.md" title = "Suggest an edit" aria-label = "Suggest an edit" >
< i id = "git-edit-button" class = "fa fa-edit" > < / i >
< / a >
< / div >
< / div >
< div id = "search-wrapper" class = "hidden" >
< form id = "searchbar-outer" class = "searchbar-outer" >
< input type = "search" id = "searchbar" name = "searchbar" placeholder = "Search this book ..." aria-controls = "searchresults-outer" aria-describedby = "searchresults-header" >
< / form >
< div id = "searchresults-outer" class = "searchresults-outer hidden" >
< div id = "searchresults-header" class = "searchresults-header" > < / div >
< ul id = "searchresults" >
< / ul >
< / div >
< / div >
<!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
< script type = "text/javascript" >
document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
});
< / script >
< div id = "content" class = "content" >
< main >
<!-- Page table of contents -->
< div class = "sidetoc" >
< nav class = "pagetoc" > < / nav >
< / div >
< h1 id = "room-dag-concepts" > < a class = "header" href = "#room-dag-concepts" > Room DAG concepts< / a > < / h1 >
< h2 id = "edges" > < a class = "header" href = "#edges" > Edges< / a > < / h2 >
< p > The word " edge" comes from graph theory lingo. An edge is just a connection
between two events. In Synapse, we connect events by specifying their
< code > prev_events< / code > . A subsequent event points back at a previous event.< / p >
< pre > < code > A (oldest) < ---- B < ---- C (most recent)
< / code > < / pre >
< h2 id = "depth-and-stream-ordering" > < a class = "header" href = "#depth-and-stream-ordering" > Depth and stream ordering< / a > < / h2 >
< p > Events are normally sorted by < code > (topological_ordering, stream_ordering)< / code > where
< code > topological_ordering< / code > is just < code > depth< / code > . In other words, we first sort by < code > depth< / code >
and then tie-break based on < code > stream_ordering< / code > . < code > depth< / code > is incremented as new
messages are added to the DAG. Normally, < code > stream_ordering< / code > is an auto
incrementing integer, but backfilled events start with < code > stream_ordering=-1< / code > and decrement.< / p >
< hr / >
< ul >
< li > < code > /sync< / code > returns things in the order they arrive at the server (< code > stream_ordering< / code > ).< / li >
< li > < code > /messages< / code > (and < code > /backfill< / code > in the federation API) return them in the order determined by the event graph < code > (topological_ordering, stream_ordering)< / code > .< / li >
< / ul >
< p > The general idea is that, if you're following a room in real-time (i.e.
< code > /sync< / code > ), you probably want to see the messages as they arrive at your server,
rather than skipping any that arrived late; whereas if you're looking at a
historical section of timeline (i.e. < code > /messages< / code > ), you want to see the best
representation of the state of the room as others were seeing it at the time.< / p >
2022-03-22 10:30:53 -06:00
< h2 id = "outliers" > < a class = "header" href = "#outliers" > Outliers< / a > < / h2 >
< p > We mark an event as an < code > outlier< / code > when we haven't figured out the state for the
room at that point in the DAG yet. They are " floating" events that we haven't
yet correlated to the DAG.< / p >
< p > Outliers typically arise when we fetch the auth chain or state for a given
event. When that happens, we just grab the events in the state/auth chain,
without calculating the state at those events, or backfilling their
2022-04-19 06:25:04 -06:00
< code > prev_events< / code > . Since we don't have the state at any events fetched in that
way, we mark them as outliers.< / p >
2022-03-22 10:30:53 -06:00
< p > So, typically, we won't have the < code > prev_events< / code > of an < code > outlier< / code > in the database,
(though it's entirely possible that we < em > might< / em > have them for some other
reason). Other things that make outliers different from regular events:< / p >
< ul >
< li >
< p > We don't have state for them, so there should be no entry in
< code > event_to_state_groups< / code > for an outlier. (In practice this isn't always
the case, though I'm not sure why: see https://github.com/matrix-org/synapse/issues/12201).< / p >
< / li >
< li >
< p > We don't record entries for them in the < code > event_edges< / code > ,
< code > event_forward_extremeties< / code > or < code > event_backward_extremities< / code > tables.< / p >
< / li >
< / ul >
< p > Since outliers are not tied into the DAG, they do not normally form part of the
timeline sent down to clients via < code > /sync< / code > or < code > /messages< / code > ; however there is an
exception:< / p >
< h3 id = "out-of-band-membership-events" > < a class = "header" href = "#out-of-band-membership-events" > Out-of-band membership events< / a > < / h3 >
< p > A special case of outlier events are some membership events for federated rooms
that we aren't full members of. For example:< / p >
< ul >
< li > invites received over federation, before we join the room< / li >
< li > < em > rejections< / em > for said invites< / li >
< li > knock events for rooms that we would like to join but have not yet joined.< / li >
< / ul >
< p > In all the above cases, we don't have the state for the room, which is why they
are treated as outliers. They are a bit special though, in that they are
proactively sent to clients via < code > /sync< / code > .< / p >
2021-08-10 07:24:16 -06:00
< h2 id = "forward-extremity" > < a class = "header" href = "#forward-extremity" > Forward extremity< / a > < / h2 >
2022-03-22 10:30:53 -06:00
< p > Most-recent-in-time events in the DAG which are not referenced by any other
events' < code > prev_events< / code > yet. (In this definition, outliers, rejected events, and
soft-failed events don't count.)< / p >
< p > The forward extremities of a room (or at least, a subset of them, if there are
more than ten) are used as the < code > prev_events< / code > when the next event is sent.< / p >
< p > The " current state" of a room (ie: the state which would be used if we
generated a new event) is, therefore, the resolution of the room states
at each of the forward extremities.< / p >
2021-12-14 07:23:10 -07:00
< h2 id = "backward-extremity" > < a class = "header" href = "#backward-extremity" > Backward extremity< / a > < / h2 >
2021-08-10 07:24:16 -06:00
< p > The current marker of where we have backfilled up to and will generally be the
2021-12-14 07:23:10 -07:00
< code > prev_events< / code > of the oldest-in-time events we have in the DAG. This gives a starting point when
backfilling history.< / p >
2022-03-22 10:30:53 -06:00
< p > Note that, unlike forward extremities, we typically don't have any backward
extremity events themselves in the database - or, if we do, they will be " outliers" (see
above). Either way, we don't expect to have the room state at a backward extremity.< / p >
< p > When we persist a non-outlier event, if it was previously a backward extremity,
we clear it as a backward extremity and set all of its < code > prev_events< / code > as the new
backward extremities if they aren't already persisted as non-outliers. This
therefore keeps the backward extremities up-to-date.< / p >
2021-08-10 07:24:16 -06:00
< h2 id = "state-groups" > < a class = "header" href = "#state-groups" > State groups< / a > < / h2 >
< p > For every non-outlier event we need to know the state at that event. Instead of
storing the full state for each event in the DB (i.e. a < code > event_id -> state< / code >
mapping), which is < em > very< / em > space inefficient when state doesn't change, we
instead assign each different set of state a " state group" and then have
mappings of < code > event_id -> state_group< / code > and < code > state_group -> state< / code > .< / p >
< h3 id = "stage-group-edges" > < a class = "header" href = "#stage-group-edges" > Stage group edges< / a > < / h3 >
< p > TODO: < code > state_group_edges< / code > is a further optimization...
notes from @Azrenbeth, https://pastebin.com/seUGVGeT< / p >
< / main >
< nav class = "nav-wrapper" aria-label = "Page navigation" >
<!-- Mobile navigation buttons -->
< a rel = "prev" href = "../development/cas.html" class = "mobile-nav-chapters previous" title = "Previous chapter" aria-label = "Previous chapter" aria-keyshortcuts = "Left" >
< i class = "fa fa-angle-left" > < / i >
< / a >
< a rel = "next" href = "../auth_chain_difference_algorithm.html" class = "mobile-nav-chapters next" title = "Next chapter" aria-label = "Next chapter" aria-keyshortcuts = "Right" >
< i class = "fa fa-angle-right" > < / i >
< / a >
< div style = "clear: both" > < / div >
< / nav >
< / div >
< / div >
< nav class = "nav-wide-wrapper" aria-label = "Page navigation" >
< a rel = "prev" href = "../development/cas.html" class = "nav-chapters previous" title = "Previous chapter" aria-label = "Previous chapter" aria-keyshortcuts = "Left" >
< i class = "fa fa-angle-left" > < / i >
< / a >
< a rel = "next" href = "../auth_chain_difference_algorithm.html" class = "nav-chapters next" title = "Next chapter" aria-label = "Next chapter" aria-keyshortcuts = "Right" >
< i class = "fa fa-angle-right" > < / i >
< / a >
< / nav >
< / div >
< script type = "text/javascript" >
window.playground_copyable = true;
< / script >
< script src = "../elasticlunr.min.js" type = "text/javascript" charset = "utf-8" > < / script >
< script src = "../mark.min.js" type = "text/javascript" charset = "utf-8" > < / script >
< script src = "../searcher.js" type = "text/javascript" charset = "utf-8" > < / script >
< script src = "../clipboard.min.js" type = "text/javascript" charset = "utf-8" > < / script >
< script src = "../highlight.js" type = "text/javascript" charset = "utf-8" > < / script >
< script src = "../book.js" type = "text/javascript" charset = "utf-8" > < / script >
<!-- Custom JS scripts -->
< script type = "text/javascript" src = "../docs/website_files/table-of-contents.js" > < / script >
< / body >
< / html >
