diff --git a/.gitignore b/.gitignore index af90668c89..3899c5a250 100644 --- a/.gitignore +++ b/.gitignore @@ -26,17 +26,18 @@ htmlcov demo/*.db demo/*.log +demo/*.log.* demo/*.pid +demo/media_store.* demo/etc -graph/*.svg -graph/*.png -graph/*.dot - -**/webclient/config.js -**/webclient/test/coverage/ -**/webclient/test/environment-protractor.js - uploads .idea/ +media_store/ + +*.tac + +build/ + +localhost-800*/ diff --git a/CHANGES.rst b/CHANGES.rst index 297ae914fd..8835107594 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,54 +1,78 @@ +Changes in synapse v0.7.0 (2015-02-12) +====================================== + +* Add initial implementation of the query auth federation API, allowing + servers to agree on whether an event should be allowed or rejected. +* Persist events we have rejected from federation, fixing the bug where + servers would keep requesting the same events. +* Various federation performance improvements, including: + + - Add in memory caches on queries such as: + + * Computing the state of a room at a point in time, used for + authorization on federation requests. + * Fetching events from the database. + * User's room membership, used for authorizing presence updates. + + - Upgraded JSON library to improve parsing and serialisation speeds. + +* Add default avatars to new user accounts using pydenticon library. +* Correctly time out federation requests. +* Retry federation requests against different servers. +* Add support for push and push rules. +* Add alpha versions of proposed new CSv2 APIs, including ``/sync`` API. + Changes in synapse 0.6.1 (2015-01-07) ===================================== - * Major optimizations to improve performance of initial sync and event sending - in large rooms (by up to 10x) - * Media repository now includes a Content-Length header on media downloads. - * Improve quality of thumbnails by changing resizing algorithm. +* Major optimizations to improve performance of initial sync and event sending + in large rooms (by up to 10x) +* Media repository now includes a Content-Length header on media downloads. +* Improve quality of thumbnails by changing resizing algorithm. Changes in synapse 0.6.0 (2014-12-16) ===================================== - * Add new API for media upload and download that supports thumbnailing. - * Replicate media uploads over multiple homeservers so media is always served - to clients from their local homeserver. This obsoletes the - --content-addr parameter and confusion over accessing content directly - from remote homeservers. - * Implement exponential backoff when retrying federation requests when - sending to remote homeservers which are offline. - * Implement typing notifications. - * Fix bugs where we sent events with invalid signatures due to bugs where - we incorrectly persisted events. - * Improve performance of database queries involving retrieving events. +* Add new API for media upload and download that supports thumbnailing. +* Replicate media uploads over multiple homeservers so media is always served + to clients from their local homeserver. This obsoletes the + --content-addr parameter and confusion over accessing content directly + from remote homeservers. +* Implement exponential backoff when retrying federation requests when + sending to remote homeservers which are offline. +* Implement typing notifications. +* Fix bugs where we sent events with invalid signatures due to bugs where + we incorrectly persisted events. +* Improve performance of database queries involving retrieving events. Changes in synapse 0.5.4a (2014-12-13) ====================================== - * Fix bug while generating the error message when a file path specified in - the config doesn't exist. +* Fix bug while generating the error message when a file path specified in + the config doesn't exist. Changes in synapse 0.5.4 (2014-12-03) ===================================== - * Fix presence bug where some rooms did not display presence updates for - remote users. - * Do not log SQL timing log lines when started with "-v" - * Fix potential memory leak. +* Fix presence bug where some rooms did not display presence updates for + remote users. +* Do not log SQL timing log lines when started with "-v" +* Fix potential memory leak. Changes in synapse 0.5.3c (2014-12-02) ====================================== - * Change the default value for the `content_addr` option to use the HTTP - listener, as by default the HTTPS listener will be using a self-signed - certificate. +* Change the default value for the `content_addr` option to use the HTTP + listener, as by default the HTTPS listener will be using a self-signed + certificate. Changes in synapse 0.5.3 (2014-11-27) ===================================== - * Fix bug that caused joining a remote room to fail if a single event was not - signed correctly. - * Fix bug which caused servers to continuously try and fetch events from other - servers. +* Fix bug that caused joining a remote room to fail if a single event was not + signed correctly. +* Fix bug which caused servers to continuously try and fetch events from other + servers. Changes in synapse 0.5.2 (2014-11-26) ===================================== diff --git a/MANIFEST.in b/MANIFEST.in index a1a77ff540..8243a942ee 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,14 @@ -recursive-include docs * -recursive-include tests *.py +include synctl +include LICENSE +include VERSION +include *.rst +include demo/README + recursive-include synapse/storage/schema *.sql -recursive-include syweb/webclient * + +recursive-include demo *.dh +recursive-include demo *.py +recursive-include demo *.sh +recursive-include docs * +recursive-include scripts * +recursive-include tests *.py diff --git a/README.rst b/README.rst index e1fa4d75ff..5eebe5b721 100644 --- a/README.rst +++ b/README.rst @@ -96,6 +96,11 @@ Installing prerequisites on Ubuntu or Debian:: $ sudo apt-get install build-essential python2.7-dev libffi-dev \ python-pip python-setuptools sqlite3 \ libssl-dev python-virtualenv libjpeg-dev + +Installing prerequisites on ArchLinux:: + + $ sudo pacman -S base-devel python2 python-pip \ + python-setuptools python-virtualenv sqlite3 Installing prerequisites on Mac OS X:: @@ -148,6 +153,39 @@ failing, e.g.:: On OSX, if you encounter clang: error: unknown argument: '-mno-fused-madd' you will need to export CFLAGS=-Qunused-arguments. +ArchLinux +--------- + +Installation on ArchLinux may encounter a few hiccups as Arch defaults to +python 3, but synapse currently assumes python 2.7 by default. + +pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):: + + $ sudo pip2.7 install --upgrade pip + +You also may need to explicitly specify python 2.7 again during the install +request:: + + $ pip2.7 install --process-dependency-links \ + https://github.com/matrix-org/synapse/tarball/master + +If you encounter an error with lib bcrypt causing an Wrong ELF Class: +ELFCLASS32 (x64 Systems), you may need to reinstall py-bcrypt to correctly +compile it under the right architecture. (This should not be needed if +installing under virtualenv):: + + $ sudo pip2.7 uninstall py-bcrypt + $ sudo pip2.7 install py-bcrypt + +During setup of homeserver you need to call python2.7 directly again:: + + $ python2.7 -m synapse.app.homeserver \ + --server-name machine.my.domain.name \ + --config-path homeserver.yaml \ + --generate-config + +...substituting your host and domain name as appropriate. + Windows Install --------------- Synapse can be installed on Cygwin. It requires the following Cygwin packages: @@ -207,6 +245,14 @@ fix try re-installing from PyPI or directly from $ # Install from github $ pip install --user https://github.com/pyca/pynacl/tarball/master +ArchLinux +--------- + +If running `$ synctl start` fails wit 'returned non-zero exit status 1', you will need to explicitly call Python2.7 - either running as:: + + $ python2.7 -m synapse.app.homeserver --daemonize -c homeserver.yaml --pid-file homeserver.pid + +...or by editing synctl with the correct python executable. Homeserver Development ====================== diff --git a/UPGRADE.rst b/UPGRADE.rst index 9618ad2d57..6ea348acc3 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -1,3 +1,17 @@ +Upgrading to v0.7.0 +=================== + +New dependencies are: + +- pydenticon +- simplejson +- syutil +- matrix-angular-sdk + +To pull in these dependencies in a virtual env, run:: + + python synapse/python_dependencies.py | xargs -n 1 pip install + Upgrading to v0.6.0 =================== @@ -52,7 +66,7 @@ resulting conflicts during the upgrade process. Before running the command the homeserver should be first completely shutdown. To run it, simply specify the location of the database, e.g.: - ./database-prepare-for-0.5.0.sh "homeserver.db" + ./scripts/database-prepare-for-0.5.0.sh "homeserver.db" Once this has successfully completed it will be safe to restart the homeserver. You may notice that the homeserver takes a few seconds longer to @@ -147,7 +161,7 @@ rooms the home server was a member of and room alias mappings. Before running the command the homeserver should be first completely shutdown. To run it, simply specify the location of the database, e.g.: - ./database-prepare-for-0.0.1.sh "homeserver.db" + ./scripts/database-prepare-for-0.0.1.sh "homeserver.db" Once this has successfully completed it will be safe to restart the homeserver. You may notice that the homeserver takes a few seconds longer to diff --git a/VERSION b/VERSION deleted file mode 100644 index f97571ce52..0000000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.6.1f diff --git a/cmdclient/console.py b/contrib/cmdclient/console.py similarity index 100% rename from cmdclient/console.py rename to contrib/cmdclient/console.py diff --git a/cmdclient/http.py b/contrib/cmdclient/http.py similarity index 100% rename from cmdclient/http.py rename to contrib/cmdclient/http.py diff --git a/experiments/cursesio.py b/contrib/experiments/cursesio.py similarity index 100% rename from experiments/cursesio.py rename to contrib/experiments/cursesio.py diff --git a/experiments/test_messaging.py b/contrib/experiments/test_messaging.py similarity index 100% rename from experiments/test_messaging.py rename to contrib/experiments/test_messaging.py diff --git a/graph/graph.py b/contrib/graph/graph.py similarity index 100% rename from graph/graph.py rename to contrib/graph/graph.py diff --git a/graph/graph2.py b/contrib/graph/graph2.py similarity index 84% rename from graph/graph2.py rename to contrib/graph/graph2.py index b9b8a562a0..6b551d42e5 100644 --- a/graph/graph2.py +++ b/contrib/graph/graph2.py @@ -23,14 +23,27 @@ import argparse from synapse.events import FrozenEvent -def make_graph(db_name, room_id, file_prefix): +def make_graph(db_name, room_id, file_prefix, limit): conn = sqlite3.connect(db_name) - c = conn.execute( - "SELECT json FROM event_json where room_id = ?", - (room_id,) + sql = ( + "SELECT json FROM event_json as j " + "INNER JOIN events as e ON e.event_id = j.event_id " + "WHERE j.room_id = ?" ) + args = [room_id] + + if limit: + sql += ( + " ORDER BY topological_ordering DESC, stream_ordering DESC " + "LIMIT ?" + ) + + args.append(limit) + + c = conn.execute(sql, args) + events = [FrozenEvent(json.loads(e[0])) for e in c.fetchall()] events.sort(key=lambda e: e.depth) @@ -128,11 +141,16 @@ if __name__ == "__main__": ) parser.add_argument( "-p", "--prefix", dest="prefix", - help="String to prefix output files with" + help="String to prefix output files with", + default="graph_output" + ) + parser.add_argument( + "-l", "--limit", + help="Only retrieve the last N events.", ) parser.add_argument('db') parser.add_argument('room') args = parser.parse_args() - make_graph(args.db, args.room, args.prefix) + make_graph(args.db, args.room, args.prefix, args.limit) diff --git a/contrib/jitsimeetbridge/jitsimeetbridge.py b/contrib/jitsimeetbridge/jitsimeetbridge.py index dbc6f6ffa5..15f8e1c48b 100644 --- a/contrib/jitsimeetbridge/jitsimeetbridge.py +++ b/contrib/jitsimeetbridge/jitsimeetbridge.py @@ -39,43 +39,43 @@ ROOMDOMAIN="meet.jit.si" #ROOMDOMAIN="conference.jitsi.vuc.me" class TrivialMatrixClient: - def __init__(self, access_token): - self.token = None - self.access_token = access_token + def __init__(self, access_token): + self.token = None + self.access_token = access_token - def getEvent(self): - while True: - url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" - if self.token: - url += "&from="+self.token - req = grequests.get(url) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "incoming from matrix",obj - if 'end' not in obj: - continue - self.token = obj['end'] - if len(obj['chunk']): - return obj['chunk'][0] + def getEvent(self): + while True: + url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000" + if self.token: + url += "&from="+self.token + req = grequests.get(url) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "incoming from matrix",obj + if 'end' not in obj: + continue + self.token = obj['end'] + if len(obj['chunk']): + return obj['chunk'][0] - def joinRoom(self, roomId): - url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token - print url - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data='{}') - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def joinRoom(self, roomId): + url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token + print url + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data='{}') + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj - def sendEvent(self, roomId, evType, event): - url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token - print url - print json.dumps(event) - headers={ 'Content-Type': 'application/json' } - req = grequests.post(url, headers=headers, data=json.dumps(event)) - resps = grequests.map([req]) - obj = json.loads(resps[0].content) - print "response: ",obj + def sendEvent(self, roomId, evType, event): + url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token + print url + print json.dumps(event) + headers={ 'Content-Type': 'application/json' } + req = grequests.post(url, headers=headers, data=json.dumps(event)) + resps = grequests.map([req]) + obj = json.loads(resps[0].content) + print "response: ",obj @@ -83,178 +83,178 @@ xmppClients = {} def matrixLoop(): - while True: - ev = matrixCli.getEvent() - print ev - if ev['type'] == 'm.room.member': - print 'membership event' - if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: - roomId = ev['room_id'] - print "joining room %s" % (roomId) - matrixCli.joinRoom(roomId) - elif ev['type'] == 'm.room.message': - if ev['room_id'] in xmppClients: - print "already have a bridge for that user, ignoring" - continue - print "got message, connecting" - xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.invite': - print "Incoming call" - #sdp = ev['content']['offer']['sdp'] - #print "sdp: %s" % (sdp) - #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) - #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) - elif ev['type'] == 'm.call.answer': - print "Call answered" - sdp = ev['content']['answer']['sdp'] - if ev['room_id'] not in xmppClients: - print "We didn't have a call for that room" - continue - # should probably check call ID too - xmppCli = xmppClients[ev['room_id']] - xmppCli.sendAnswer(sdp) - elif ev['type'] == 'm.call.hangup': - if ev['room_id'] in xmppClients: - xmppClients[ev['room_id']].stop() - del xmppClients[ev['room_id']] - + while True: + ev = matrixCli.getEvent() + print ev + if ev['type'] == 'm.room.member': + print 'membership event' + if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME: + roomId = ev['room_id'] + print "joining room %s" % (roomId) + matrixCli.joinRoom(roomId) + elif ev['type'] == 'm.room.message': + if ev['room_id'] in xmppClients: + print "already have a bridge for that user, ignoring" + continue + print "got message, connecting" + xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.invite': + print "Incoming call" + #sdp = ev['content']['offer']['sdp'] + #print "sdp: %s" % (sdp) + #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id']) + #gevent.spawn(xmppClients[ev['room_id']].xmppLoop) + elif ev['type'] == 'm.call.answer': + print "Call answered" + sdp = ev['content']['answer']['sdp'] + if ev['room_id'] not in xmppClients: + print "We didn't have a call for that room" + continue + # should probably check call ID too + xmppCli = xmppClients[ev['room_id']] + xmppCli.sendAnswer(sdp) + elif ev['type'] == 'm.call.hangup': + if ev['room_id'] in xmppClients: + xmppClients[ev['room_id']].stop() + del xmppClients[ev['room_id']] + class TrivialXmppClient: - def __init__(self, matrixRoom, userId): - self.rid = 0 - self.matrixRoom = matrixRoom - self.userId = userId - self.running = True + def __init__(self, matrixRoom, userId): + self.rid = 0 + self.matrixRoom = matrixRoom + self.userId = userId + self.running = True - def stop(self): - self.running = False + def stop(self): + self.running = False - def nextRid(self): - self.rid += 1 - return '%d' % (self.rid) + def nextRid(self): + self.rid += 1 + return '%d' % (self.rid) - def sendIq(self, xml): - fullXml = "
%s" % (self.nextRid(), self.sid, xml) - #print "\t>>>%s" % (fullXml) - return self.xmppPoke(fullXml) - - def xmppPoke(self, xml): - headers = {'Content-Type': 'application/xml'} - req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) - resps = grequests.map([req]) - obj = BeautifulSoup(resps[0].content) - return obj + def sendIq(self, xml): + fullXml = "%s" % (self.nextRid(), self.sid, xml) + #print "\t>>>%s" % (fullXml) + return self.xmppPoke(fullXml) - def sendAnswer(self, answer): - print "sdp from matrix client",answer - p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - jingle, out_err = p.communicate(answer) - jingle = jingle % { - 'tojid': self.callfrom, - 'action': 'session-accept', - 'initiator': self.callfrom, - 'responder': self.jid, - 'sid': self.callsid - } - print "answer jingle from sdp",jingle - res = self.sendIq(jingle) - print "reply from answer: ",res - - self.ssrcs = {} - jingleSoup = BeautifulSoup(jingle) - for cont in jingleSoup.iq.jingle.findAll('content'): - if cont.description: - self.ssrcs[cont['name']] = cont.description['ssrc'] - print "my ssrcs:",self.ssrcs + def xmppPoke(self, xml): + headers = {'Content-Type': 'application/xml'} + req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml) + resps = grequests.map([req]) + obj = BeautifulSoup(resps[0].content) + return obj - gevent.joinall([ - gevent.spawn(self.advertiseSsrcs) - ]) - - def advertiseSsrcs(self): + def sendAnswer(self, answer): + print "sdp from matrix client",answer + p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE) + jingle, out_err = p.communicate(answer) + jingle = jingle % { + 'tojid': self.callfrom, + 'action': 'session-accept', + 'initiator': self.callfrom, + 'responder': self.jid, + 'sid': self.callsid + } + print "answer jingle from sdp",jingle + res = self.sendIq(jingle) + print "reply from answer: ",res + + self.ssrcs = {} + jingleSoup = BeautifulSoup(jingle) + for cont in jingleSoup.iq.jingle.findAll('content'): + if cont.description: + self.ssrcs[cont['name']] = cont.description['ssrc'] + print "my ssrcs:",self.ssrcs + + gevent.joinall([ + gevent.spawn(self.advertiseSsrcs) + ]) + + def advertiseSsrcs(self): time.sleep(7) - print "SSRC spammer started" - while self.running: - ssrcMsg = "