Mirrors
/
synapse
mirror of https://github.com/element-hq/synapse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Write the upload portion of version 1 of the media repository

This commit is contained in:
Mark Haines 2014-12-02 15:09:51 +00:00
parent 15099fade5
commit 279c48c8b4
8 changed files with 451 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/media_repository.rst Normal file
View File

@ -0,0 +1,20 @@
Media Repository
================
The media repository is where attachments and avatar photos are stored.
It stores attachment content and thumbnails for media uploaded by local users.
It caches attachment content and thumbnails for media uploaded by remote users.
Storage
-------
Each item of media is assigned a ``media_id`` when it is uploaded.
The ``media_id`` is a randomly chosen, URL safe 24 character string.
Metadata such as the MIME type, upload time and length are stored in the
sqlite3 database indexed by ``media_id``.
Content is stored on the filesystem under a "content" directory. Thumbnails are
stored under a "thumbnails" directory.
The item with ``media_id`` ``"aabbccccccccdddddddddddd"`` is stored under
``"local/content/aa/bb/ccccccccdddddddddddd"``. Its thumbnail with width
``128`` and height ``96`` and type ``"image/jpeg"`` is stored under
``"local/thumbnails/aa/bb/ccccccccdddddddddddd/128-96-image-jpeg"``

21
synapse/http/server.py
View File

@ -166,14 +166,10 @@ class JsonResource(HttpServer, resource.Resource):
request)
return
if not self._request_user_agent_is_curl(request):
json_bytes = encode_canonical_json(response_json_object)
else:
json_bytes = encode_pretty_printed_json(response_json_object)
# TODO: Only enable CORS for the requests that need it.
respond_with_json_bytes(request, code, json_bytes, send_cors=True,
response_code_message=response_code_message)
respond_with_json(request, code, response_json_object, send_cors=True,
response_code_message=response_code_message,
pretty_print=self._request_user_agent_is_curl)
@staticmethod
def _request_user_agent_is_curl(request):
@ -202,6 +198,17 @@ class RootRedirect(resource.Resource):
return resource.Resource.getChild(self, name, request)
def respond_with_json(request, code, json_object, send_cors=False,
response_code_message=None, pretty_print=False):
if not pretty_print:
json_bytes = encode_pretty_printed_json(response_json_object)
else:
json_bytes = encode_canonical_json(response_json_object)
return respond_with_json_bytes(request, code, json_bytes, send_cors,
response_code_message=response_code_message)
def respond_with_json_bytes(request, code, json_bytes, send_cors=False,
response_code_message=None):
"""Sends encoded JSON in response to the given request.

0
synapse/http/content_repository.py → synapse/media/v0/content_repository.py
View File

53
synapse/media/v1/filepath.py Normal file
View File

@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
class MediaFilePaths(object):
def __init__(self, base_path):
self.base_path = base_path
def local_media_filepath(self, media_id):
return os.path.join(
self.base_path, "local", "content",
media_id[0:2], media_id[2:4], media_id[4:]
)
def local_media_thumbnail(self, media_id, width, height, content_type):
top_level_type, sub_type = content_type.split("/")
file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
return os.path.join(
self.base_path, "local", "thumbnails",
media_id[0:2], media_id[2:4], media_id[4:],
file_name
)
def remote_media_filepath(self, server_name, file_id):
return os.path.join(
self.base_path, "remote", "content", server_name,
file_id[0:2], file_id[2:4], file_id[4:]
)
def remote_media_thumbnail(self, server_name, file_id, width, height,
content_type):
top_level_type, sub_type = content_type.split("/")
file_name = "%i-%i-%s-%s" % (width, height, top_level_type, sub_type)
return os.path.join(
self.base_path, "remote", "content", server_name,
file_id[0:2], file_id[2:4], file_id[4:],
file_name
)

72
synapse/media/v1/media_repository.py Normal file
View File

@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.http.server import respond_with_json_bytes
from synapse.util.stringutils import random_string
from synapse.api.errors import (
cs_exception, SynapseError, CodeMessageException, Codes, cs_error
)
from twisted.protocols.basic import FileSender
from twisted.web import server, resource
from twisted.internet import defer
import base64
import json
import logging
import os
import re
logger = logging.getLogger(__name__)
class MediaRepository():
"""Profiles file uploading and downloading.
Uploads are POSTed to a resource which returns a token which is used to GET
the download::
=> POST /_matrix/media/v1/upload HTTP/1.1
Content-Type: <media-type>
<media>
<= HTTP/1.1 200 OK
Content-Type: application/json
{ "token": <media-id> }
=> GET /_matrix/media/v1/download/<media-id> HTTP/1.1
<= HTTP/1.1 200 OK
Content-Type: <media-type>
Content-Disposition: attachment;filename=<upload-filename>
<media>
Clients can get thumbnails by supplying a desired width and height::
=> GET /_matrix/media/v1/thumbnail/<media-id>?width=<w>&height=<h> HTTP/1.1
<= HTTP/1.1 200 OK
Content-Type: image/jpeg or image/png
<thumbnail>
"""
def __init__(self, hs):
filepaths = MediaFilePaths

110
synapse/media/v1/upload_resource.py Normal file
View File

@ -0,0 +1,110 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from synapse.http.server import respond_with_json
from synapse.util.stringutils import random_string
from synapse.api.errors import (
cs_exception, SynapseError, CodeMessageException
)
from twisted.web import server, resource
from twisted.internet import defer
import logging
logger = logging.getLogger(__name__)
class UploadResource(resource.Resource):
def __init__(self, hs, filepaths):
self.auth = hs.get_auth()
self.store = hs.get_datastore()
self.max_upload_size = hs.config.max_upload_size()
self.filepaths = filepaths
def render_POST(self, request):
self._async_render_POST(request)
return server.NOT_DONE_YET
def render_OPTIONS(self, request):
respond_with_json(request, 200, {}, send_cors=True)
return server.NOT_DONE_YET
@defer.inlineCallbacks
def _async_render_POST(self, request):
auth_user = yield self.auth.get_user_by_req(request)
try:
# TODO: The checks here are a bit late. The content will have
# already been uploaded to a tmp file at this point
content_length = request.getHeader("Content-Length")
if content_length is None:
raise SynapseError(
msg="Request must specify a Content-Length", code=400
)
if int(content_length) > self.max_upload_size:
raise SynapseError(
msg="Upload request body is too large",
code=413,
)
headers = request.requestHeaders()
if headers.hasHeader("Content-Type"):
media_type = headers.getRawHeaders("Content-Type")[0]
else:
raise SynapseError(
msg="Upload request missing 'Content-Type'",
code=400,
)
#if headers.hasHeader("Content-Disposition"):
# disposition = headers.getRawHeaders("Content-Disposition")[0]
# TODO(markjh): parse content-dispostion
media_id = random_string(24)
fname = self.filepaths.local_media_file_path(media_id)
# This shouldn't block for very long because the content will have
# already been uploaded at this point.
with open(fname, "wb") as f:
f.write(request.content.read())
yield self.store.store_local_media(
media_id=media_id,
media_type=media_type,
time_now_ms=self.clock.time_msec(),
upload_name=None,
media_length=content_length,
user_id=auth_user,
)
respond_with_json(
request, 200, {"content_token": media_id}, send_cors=True
)
except CodeMessageException as e:
logger.exception(e)
respond_with_json(request, e.code, cs_exception(e), send_cors=True)
except:
logger.exception("Failed to store file")
respond_with_json(
request,
500,
{"error": "Internal server error"},
send_cors=True
)

116
synapse/storage/media_repository.py Normal file
View File

@ -0,0 +1,116 @@
# -*- coding: utf-8 -*-
# Copyright 2014 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from _base import SQLBaseStore
class MediaRepositoryStore(SQLBaseStore):
"""Persistence for attachments and avatars"""
def get_local_media(self, media_id):
return self._simple_select_one(
"local_media_repository",
{"media_id": media_id},
("media_type", "media_length", "upload_name", "created_ts"),
)
def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
media_length, user_id):
return self._simple_insert(
"local_media_repository",
{
"media_id": media_id,
"media_type": media_type,
"created_ts": time_now_ms,
"upload_name": upload_name,
"media_length": media_length,
"user_id": user_id,
}
)
def get_local_media_thumbnails(self, media_id):
return self._simple_select_list(
"local_media_thumbnails",
{"media_id": media_id},
(
"thumbnail_width", "thumbnail_height",
"thumbnail_type", "thumbnail_length",
)
)
def store_local_thumbnail(self, media_id, thumbnail_width,
thumbnail_height, thumbnail_type,
thumbnail_length):
return self._simple_insert(
"local_media_thumbnails",
{
"media_id": media_id,
"thumbnail_width": thumbnail_width,
"thumbnail_height": thumbnail_height,
"thumbnail_type": thumbnail_type,
"thumbnail_length": thumbnail_length,
}
)
def get_cached_remote_media(self, origin, media_id):
return self._simple_select_one(
"remote_media_cache",
{"media_origin": origin, "media_id": media_id},
("media_type", "media_length", "upload_name", "created_ts"),
)
def store_cached_remote_media(self, origin, media_id, media_type,
media_length, time_now_ms, upload_name,
filesytem_id):
return self._simple_insert(
"remote_media_cache",
{
"media_origin": origin,
"media_id": media_id,
"media_type": media_type,
"media_length": media_length,
"created_ts": time_now_ms,
"upload_name": upload_name,
"filesystem_id": filesystem_id,
}
)
def get_remote_media_thumbnails(self, origin, media_id):
return self._simple_select_list(
"remote_media_cache_thumbnails",
{"origin": origin, "media_id": media_id},
(
"thumbnail_width", "thumbnail_height",
"thumbnail_type", "thumbnail_length",
"filesystem_id"
)
)
def store_remote_media_thumbnail(self, origin, media_id, thumbnail_width,
thumbnail_height, thumbnail_type,
thumbnail_length, filesystem_id):
return self._simple_insert(
"remote_media_cache_thumbnails",
{
"media_origin": origin,
"media_id": media_id,
"thumbnail_width": thumbnail_width,
"thumbnail_height": thumbnail_height,
"thumbnail_type": thumbnail_type,
"thumbnail_length": thumbnail_length,
"filesystem_id": filesystem_id,
}
)

66
synapse/storage/schema/media_repository.sql Normal file
View File

@ -0,0 +1,66 @@
/* Copyright 2014 OpenMarket Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
CREATE TABLE IF NOT EXISTS local_media_repository (
media_id TEXT, -- The id used to refer to the media.
media_type TEXT, -- The MIME-type of the media.
media_length INTEGER, -- Length of the media in bytes.
created_ts INTEGER, -- When the content was uploaded in ms.
upload_name TEXT, -- The name the media was uploaded with.
user_id TEXT, -- The user who uploaded the file.
CONSTRAINT uniqueness UNIQUE (media_id)
);
CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
media_id TEXT, -- The id used to refer to the media.
thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
thumbnail_type TEXT, -- The MIME-type of the thumbnail.
thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
CONSTRAINT uniqueness UNIQUE (
media_id, thumbnail_width, thumbnail_height, thumbnail_type
)
);
CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
ON local_media_repository_thumbnails (media_id);
CREATE TABLE IF NOT EXISTS remote_media_cache (
media_origin TEXT, -- The remote HS the media came from.
media_id TEXT, -- The id used to refer to the media on that server.
media_type TEXT, -- The MIME-type of the media.
created_ts INTEGER, -- When the content was uploaded in ms.
upload_name TEXT, -- The name the media was uploaded with.
media_length INTEGER, -- Length of the media in bytes.
filesystem_id TEXT, -- The name used to store the media on disk.
CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
);
CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
media_origin TEXT, -- The remote HS the media came from.
media_id TEXT, -- The id used to refer to the media.
thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
thumbnail_type TEXT, -- The MIME-type of the thumbnail.
thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
filesystem_id TEXT, -- The name used to store the media on disk.
CONSTRAINT uniqueness UNIQUE (
media_origin, media_id, thumbnail_width, thumbnail_height,
thumbnail_type
)
);
CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
ON local_media_repository_thumbnails (media_id);