Mirrors
/
synapse-old
mirror of https://github.com/matrix-org/synapse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Drop support for Postgres 10 in full text search code. (#14397)

This commit is contained in:
Patrick Cloke 2022-11-09 09:55:34 -05:00 committed by GitHub
parent 21447c9102
commit e9a4343cb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 42 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/14397.removal Normal file
View File

@ -0,0 +1 @@
Remove support for PostgreSQL 10.

38
synapse/storage/databases/main/search.py
View File

@ -463,18 +463,17 @@ class SearchStore(SearchBackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine): if isinstance(self.database_engine, PostgresEngine):
search_query = search_term search_query = search_term
tsquery_func = self.database_engine.tsquery_func sql = """
sql = f""" SELECT ts_rank_cd(vector, websearch_to_tsquery('english', ?)) AS rank,
SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) AS rank,
room_id, event_id room_id, event_id
FROM event_search FROM event_search
WHERE vector @@ {tsquery_func}('english', ?) WHERE vector @@ websearch_to_tsquery('english', ?)
""" """
args = [search_query, search_query] + args args = [search_query, search_query] + args
count_sql = f""" count_sql = """
SELECT room_id, count(*) as count FROM event_search SELECT room_id, count(*) as count FROM event_search
WHERE vector @@ {tsquery_func}('english', ?) WHERE vector @@ websearch_to_tsquery('english', ?)
""" """
count_args = [search_query] + count_args count_args = [search_query] + count_args
elif isinstance(self.database_engine, Sqlite3Engine): elif isinstance(self.database_engine, Sqlite3Engine):
@ -523,9 +522,7 @@ class SearchStore(SearchBackgroundUpdateStore):
highlights = None highlights = None
if isinstance(self.database_engine, PostgresEngine): if isinstance(self.database_engine, PostgresEngine):
highlights = await self._find_highlights_in_postgres( highlights = await self._find_highlights_in_postgres(search_query, events)
search_query, events, tsquery_func
)
count_sql += " GROUP BY room_id" count_sql += " GROUP BY room_id"
@ -604,18 +601,17 @@ class SearchStore(SearchBackgroundUpdateStore):
if isinstance(self.database_engine, PostgresEngine): if isinstance(self.database_engine, PostgresEngine):
search_query = search_term search_query = search_term
tsquery_func = self.database_engine.tsquery_func sql = """
sql = f""" SELECT ts_rank_cd(vector, websearch_to_tsquery('english', ?)) as rank,
SELECT ts_rank_cd(vector, {tsquery_func}('english', ?)) as rank,
origin_server_ts, stream_ordering, room_id, event_id origin_server_ts, stream_ordering, room_id, event_id
FROM event_search FROM event_search
WHERE vector @@ {tsquery_func}('english', ?) AND WHERE vector @@ websearch_to_tsquery('english', ?) AND
""" """
args = [search_query, search_query] + args args = [search_query, search_query] + args
count_sql = f""" count_sql = """
SELECT room_id, count(*) as count FROM event_search SELECT room_id, count(*) as count FROM event_search
WHERE vector @@ {tsquery_func}('english', ?) AND WHERE vector @@ websearch_to_tsquery('english', ?) AND
""" """
count_args = [search_query] + count_args count_args = [search_query] + count_args
elif isinstance(self.database_engine, Sqlite3Engine): elif isinstance(self.database_engine, Sqlite3Engine):
@ -686,9 +682,7 @@ class SearchStore(SearchBackgroundUpdateStore):
highlights = None highlights = None
if isinstance(self.database_engine, PostgresEngine): if isinstance(self.database_engine, PostgresEngine):
highlights = await self._find_highlights_in_postgres( highlights = await self._find_highlights_in_postgres(search_query, events)
search_query, events, tsquery_func
)
count_sql += " GROUP BY room_id" count_sql += " GROUP BY room_id"
@ -714,7 +708,7 @@ class SearchStore(SearchBackgroundUpdateStore):
} }
async def _find_highlights_in_postgres( async def _find_highlights_in_postgres(
self, search_query: str, events: List[EventBase], tsquery_func: str self, search_query: str, events: List[EventBase]
) -> Set[str]: ) -> Set[str]:
"""Given a list of events and a search term, return a list of words """Given a list of events and a search term, return a list of words
that match from the content of the event. that match from the content of the event.
@ -725,7 +719,6 @@ class SearchStore(SearchBackgroundUpdateStore):
Args: Args:
search_query search_query
events: A list of events events: A list of events
tsquery_func: The tsquery_* function to use when making queries
Returns: Returns:
A set of strings. A set of strings.
@ -758,7 +751,9 @@ class SearchStore(SearchBackgroundUpdateStore):
while stop_sel in value: while stop_sel in value:
stop_sel += ">" stop_sel += ">"
query = f"SELECT ts_headline(?, {tsquery_func}('english', ?), %s)" % ( query = (
"SELECT ts_headline(?, websearch_to_tsquery('english', ?), %s)"
% (
_to_postgres_options( _to_postgres_options(
{ {
"StartSel": start_sel, "StartSel": start_sel,
@ -767,6 +762,7 @@ class SearchStore(SearchBackgroundUpdateStore):
} }
) )
) )
)
txn.execute(query, (value, search_query)) txn.execute(query, (value, search_query))
(headline,) = txn.fetchall()[0] (headline,) = txn.fetchall()[0]

16
synapse/storage/engines/postgres.py
View File

@ -170,22 +170,6 @@ class PostgresEngine(
"""Do we support the `RETURNING` clause in insert/update/delete?""" """Do we support the `RETURNING` clause in insert/update/delete?"""
return True return True
@property
def tsquery_func(self) -> str:
"""
Selects a tsquery_* func to use.
Ref: https://www.postgresql.org/docs/current/textsearch-controls.html
Returns:
The function name.
"""
# Postgres 11 added support for websearch_to_tsquery.
assert self._version is not None
if self._version >= 110000:
return "websearch_to_tsquery"
return "plainto_tsquery"
def is_deadlock(self, error: Exception) -> bool: def is_deadlock(self, error: Exception) -> bool:
if isinstance(error, psycopg2.DatabaseError): if isinstance(error, psycopg2.DatabaseError):
# https://www.postgresql.org/docs/current/static/errcodes-appendix.html # https://www.postgresql.org/docs/current/static/errcodes-appendix.html

71
tests/storage/test_room_search.py
View File

@ -12,9 +12,8 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from typing import List, Tuple, Union from typing import List, Tuple
from unittest.case import SkipTest from unittest.case import SkipTest
from unittest.mock import PropertyMock, patch
from twisted.test.proto_helpers import MemoryReactor from twisted.test.proto_helpers import MemoryReactor
@ -220,10 +219,8 @@ class MessageSearchTest(HomeserverTestCase):
PHRASE = "the quick brown fox jumps over the lazy dog" PHRASE = "the quick brown fox jumps over the lazy dog"
# Each entry is a search query, followed by either a boolean of whether it is # Each entry is a search query, followed by a boolean of whether it is in the phrase.
# in the phrase OR a tuple of booleans: whether it matches using websearch COMMON_CASES = [
# and using plain search.
COMMON_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [
("nope", False), ("nope", False),
("brown", True), ("brown", True),
("quick brown", True), ("quick brown", True),
@ -231,13 +228,13 @@ class MessageSearchTest(HomeserverTestCase):
("quick \t brown", True), ("quick \t brown", True),
("jump", True), ("jump", True),
("brown nope", False), ("brown nope", False),
('"brown quick"', (False, True)), ('"brown quick"', False),
('"jumps over"', True), ('"jumps over"', True),
('"quick fox"', (False, True)), ('"quick fox"', False),
("nope OR doublenope", False), ("nope OR doublenope", False),
("furphy OR fox", (True, False)), ("furphy OR fox", True),
("fox -nope", (True, False)), ("fox -nope", True),
("fox -brown", (False, True)), ("fox -brown", False),
('"fox" quick', True), ('"fox" quick', True),
('"quick brown', True), ('"quick brown', True),
('" quick "', True), ('" quick "', True),
@ -246,11 +243,11 @@ class MessageSearchTest(HomeserverTestCase):
# TODO Test non-ASCII cases. # TODO Test non-ASCII cases.
# Case that fail on SQLite. # Case that fail on SQLite.
POSTGRES_CASES: List[Tuple[str, Union[bool, Tuple[bool, bool]]]] = [ POSTGRES_CASES = [
# SQLite treats NOT as a binary operator. # SQLite treats NOT as a binary operator.
("- fox", (False, True)), ("- fox", False),
("- nope", (True, False)), ("- nope", True),
('"-fox quick', (False, True)), ('"-fox quick', False),
# PostgreSQL skips stop words. # PostgreSQL skips stop words.
('"the quick brown"', True), ('"the quick brown"', True),
('"over lazy"', True), ('"over lazy"', True),
@ -275,7 +272,7 @@ class MessageSearchTest(HomeserverTestCase):
if isinstance(main_store.database_engine, PostgresEngine): if isinstance(main_store.database_engine, PostgresEngine):
assert main_store.database_engine._version is not None assert main_store.database_engine._version is not None
found = main_store.database_engine._version < 140000 found = main_store.database_engine._version < 140000
self.COMMON_CASES.append(('"fox quick', (found, True))) self.COMMON_CASES.append(('"fox quick', found))
def test_tokenize_query(self) -> None: def test_tokenize_query(self) -> None:
"""Test the custom logic to tokenize a user's query.""" """Test the custom logic to tokenize a user's query."""
@ -315,16 +312,10 @@ class MessageSearchTest(HomeserverTestCase):
) )
def _check_test_cases( def _check_test_cases(
self, self, store: DataStore, cases: List[Tuple[str, bool]]
store: DataStore,
cases: List[Tuple[str, Union[bool, Tuple[bool, bool]]]],
index=0,
) -> None: ) -> None:
# Run all the test cases versus search_msgs # Run all the test cases versus search_msgs
for query, expect_to_contain in cases: for query, expect_to_contain in cases:
if isinstance(expect_to_contain, tuple):
expect_to_contain = expect_to_contain[index]
result = self.get_success( result = self.get_success(
store.search_msgs([self.room_id], query, ["content.body"]) store.search_msgs([self.room_id], query, ["content.body"])
) )
@ -343,9 +334,6 @@ class MessageSearchTest(HomeserverTestCase):
# Run them again versus search_rooms # Run them again versus search_rooms
for query, expect_to_contain in cases: for query, expect_to_contain in cases:
if isinstance(expect_to_contain, tuple):
expect_to_contain = expect_to_contain[index]
result = self.get_success( result = self.get_success(
store.search_rooms([self.room_id], query, ["content.body"], 10) store.search_rooms([self.room_id], query, ["content.body"], 10)
) )
@ -366,38 +354,15 @@ class MessageSearchTest(HomeserverTestCase):
""" """
Test searching for phrases using typical web search syntax, as per postgres' websearch_to_tsquery. Test searching for phrases using typical web search syntax, as per postgres' websearch_to_tsquery.
This test is skipped unless the postgres instance supports websearch_to_tsquery. This test is skipped unless the postgres instance supports websearch_to_tsquery.
See https://www.postgresql.org/docs/current/textsearch-controls.html
""" """
store = self.hs.get_datastores().main store = self.hs.get_datastores().main
if not isinstance(store.database_engine, PostgresEngine): if not isinstance(store.database_engine, PostgresEngine):
raise SkipTest("Test only applies when postgres is used as the database") raise SkipTest("Test only applies when postgres is used as the database")
if store.database_engine.tsquery_func != "websearch_to_tsquery": self._check_test_cases(store, self.COMMON_CASES + self.POSTGRES_CASES)
raise SkipTest(
"Test only applies when postgres supporting websearch_to_tsquery is used as the database"
)
self._check_test_cases(store, self.COMMON_CASES + self.POSTGRES_CASES, index=0)
def test_postgres_non_web_search_for_phrase(self):
"""
Test postgres searching for phrases without using web search, which is used when websearch_to_tsquery isn't
supported by the current postgres version.
"""
store = self.hs.get_datastores().main
if not isinstance(store.database_engine, PostgresEngine):
raise SkipTest("Test only applies when postgres is used as the database")
# Patch supports_websearch_to_tsquery to always return False to ensure we're testing the plainto_tsquery path.
with patch(
"synapse.storage.engines.postgres.PostgresEngine.tsquery_func",
new_callable=PropertyMock,
) as supports_websearch_to_tsquery:
supports_websearch_to_tsquery.return_value = "plainto_tsquery"
self._check_test_cases(
store, self.COMMON_CASES + self.POSTGRES_CASES, index=1
)
def test_sqlite_search(self): def test_sqlite_search(self):
""" """
@ -407,4 +372,4 @@ class MessageSearchTest(HomeserverTestCase):
if not isinstance(store.database_engine, Sqlite3Engine): if not isinstance(store.database_engine, Sqlite3Engine):
raise SkipTest("Test only applies when sqlite is used as the database") raise SkipTest("Test only applies when sqlite is used as the database")
self._check_test_cases(store, self.COMMON_CASES, index=0) self._check_test_cases(store, self.COMMON_CASES)