Commit ccb8ac85 authored by Eugen Rochko's avatar Eugen Rochko

Make the streaming API also handle websockets (because trying to get the...

Make the streaming API also handle websockets (because trying to get the browser EventSource interface to
work flawlessly was a nightmare). WARNING: This commit makes the web UI connect to the streaming API instead
of ActionCable like before. This means that if you are upgrading, you should set that up beforehand.
parent 8c0bc130
...@@ -43,5 +43,5 @@ SMTP_FROM_ADDRESS=notifications@example.com ...@@ -43,5 +43,5 @@ SMTP_FROM_ADDRESS=notifications@example.com
# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front # Optional alias for S3 if you want to use Cloudfront or Cloudflare in front
# S3_CLOUDFRONT_HOST= # S3_CLOUDFRONT_HOST=
# Optional Firebase Cloud Messaging API key # Streaming API integration
FCM_API_KEY= # STREAMING_API_BASE_URL=
...@@ -13,4 +13,3 @@ ...@@ -13,4 +13,3 @@
//= require jquery //= require jquery
//= require jquery_ujs //= require jquery_ujs
//= require components //= require components
//= require cable
// Action Cable provides the framework to deal with WebSockets in Rails.
// You can generate new channels where WebSocket features live using the rails generate channel command.
//
//= require action_cable
//= require_self
(function() {
this.App || (this.App = {});
App.cable = ActionCable.createConsumer();
}).call(this);
...@@ -43,6 +43,7 @@ import hu from 'react-intl/locale-data/hu'; ...@@ -43,6 +43,7 @@ import hu from 'react-intl/locale-data/hu';
import uk from 'react-intl/locale-data/uk'; import uk from 'react-intl/locale-data/uk';
import getMessagesForLocale from '../locales'; import getMessagesForLocale from '../locales';
import { hydrateStore } from '../actions/store'; import { hydrateStore } from '../actions/store';
import createStream from '../stream';
const store = configureStore(); const store = configureStore();
...@@ -60,28 +61,27 @@ const Mastodon = React.createClass({ ...@@ -60,28 +61,27 @@ const Mastodon = React.createClass({
locale: React.PropTypes.string.isRequired locale: React.PropTypes.string.isRequired
}, },
componentWillMount() { componentDidMount() {
const { locale } = this.props; const { locale } = this.props;
const accessToken = store.getState().getIn(['meta', 'access_token']);
if (typeof App !== 'undefined') {
this.subscription = App.cable.subscriptions.create('TimelineChannel', { this.subscription = createStream(accessToken, 'user', {
received (data) { received (data) {
switch(data.event) { switch(data.event) {
case 'update': case 'update':
store.dispatch(updateTimeline('home', JSON.parse(data.payload))); store.dispatch(updateTimeline('home', JSON.parse(data.payload)));
break; break;
case 'delete': case 'delete':
store.dispatch(deleteFromTimelines(data.payload)); store.dispatch(deleteFromTimelines(data.payload));
break; break;
case 'notification': case 'notification':
store.dispatch(updateNotifications(JSON.parse(data.payload), getMessagesForLocale(locale), locale)); store.dispatch(updateNotifications(JSON.parse(data.payload), getMessagesForLocale(locale), locale));
break; break;
}
} }
}
}); });
}
// Desktop notifications // Desktop notifications
if (typeof window.Notification !== 'undefined' && Notification.permission === 'default') { if (typeof window.Notification !== 'undefined' && Notification.permission === 'default') {
...@@ -91,7 +91,8 @@ const Mastodon = React.createClass({ ...@@ -91,7 +91,8 @@ const Mastodon = React.createClass({
componentWillUnmount () { componentWillUnmount () {
if (typeof this.subscription !== 'undefined') { if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe(); this.subscription.close();
this.subscription = null;
} }
}, },
......
...@@ -8,45 +8,49 @@ import { ...@@ -8,45 +8,49 @@ import {
deleteFromTimelines deleteFromTimelines
} from '../../actions/timelines'; } from '../../actions/timelines';
import ColumnBackButtonSlim from '../../components/column_back_button_slim'; import ColumnBackButtonSlim from '../../components/column_back_button_slim';
import createStream from '../../stream';
const mapStateToProps = state => ({
accessToken: state.getIn(['meta', 'access_token'])
});
const HashtagTimeline = React.createClass({ const HashtagTimeline = React.createClass({
propTypes: { propTypes: {
params: React.PropTypes.object.isRequired, params: React.PropTypes.object.isRequired,
dispatch: React.PropTypes.func.isRequired dispatch: React.PropTypes.func.isRequired,
accessToken: React.PropTypes.string.isRequired
}, },
mixins: [PureRenderMixin], mixins: [PureRenderMixin],
_subscribe (dispatch, id) { _subscribe (dispatch, id) {
if (typeof App !== 'undefined') { const { accessToken } = this.props;
this.subscription = App.cable.subscriptions.create({
channel: 'HashtagChannel', this.subscription = createStream(accessToken, `hashtag&tag=${id}`, {
tag: id
}, { received (data) {
switch(data.event) {
received (data) { case 'update':
switch(data.event) { dispatch(updateTimeline('tag', JSON.parse(data.payload)));
case 'update': break;
dispatch(updateTimeline('tag', JSON.parse(data.payload))); case 'delete':
break; dispatch(deleteFromTimelines(data.payload));
case 'delete': break;
dispatch(deleteFromTimelines(data.payload));
break;
}
} }
}
}); });
}
}, },
_unsubscribe () { _unsubscribe () {
if (typeof this.subscription !== 'undefined') { if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe(); this.subscription.close();
this.subscription = null;
} }
}, },
componentWillMount () { componentDidMount () {
const { dispatch } = this.props; const { dispatch } = this.props;
const { id } = this.props.params; const { id } = this.props.params;
...@@ -79,4 +83,4 @@ const HashtagTimeline = React.createClass({ ...@@ -79,4 +83,4 @@ const HashtagTimeline = React.createClass({
}); });
export default connect()(HashtagTimeline); export default connect(mapStateToProps)(HashtagTimeline);
...@@ -9,46 +9,51 @@ import { ...@@ -9,46 +9,51 @@ import {
} from '../../actions/timelines'; } from '../../actions/timelines';
import { defineMessages, injectIntl } from 'react-intl'; import { defineMessages, injectIntl } from 'react-intl';
import ColumnBackButtonSlim from '../../components/column_back_button_slim'; import ColumnBackButtonSlim from '../../components/column_back_button_slim';
import createStream from '../../stream';
const messages = defineMessages({ const messages = defineMessages({
title: { id: 'column.public', defaultMessage: 'Public' } title: { id: 'column.public', defaultMessage: 'Public' }
}); });
const mapStateToProps = state => ({
accessToken: state.getIn(['meta', 'access_token'])
});
const PublicTimeline = React.createClass({ const PublicTimeline = React.createClass({
propTypes: { propTypes: {
dispatch: React.PropTypes.func.isRequired, dispatch: React.PropTypes.func.isRequired,
intl: React.PropTypes.object.isRequired intl: React.PropTypes.object.isRequired,
accessToken: React.PropTypes.string.isRequired
}, },
mixins: [PureRenderMixin], mixins: [PureRenderMixin],
componentWillMount () { componentDidMount () {
const { dispatch } = this.props; const { dispatch, accessToken } = this.props;
dispatch(refreshTimeline('public')); dispatch(refreshTimeline('public'));
if (typeof App !== 'undefined') { this.subscription = createStream(accessToken, 'public', {
this.subscription = App.cable.subscriptions.create('PublicChannel', {
received (data) { received (data) {
switch(data.event) { switch(data.event) {
case 'update': case 'update':
dispatch(updateTimeline('public', JSON.parse(data.payload))); dispatch(updateTimeline('public', JSON.parse(data.payload)));
break; break;
case 'delete': case 'delete':
dispatch(deleteFromTimelines(data.payload)); dispatch(deleteFromTimelines(data.payload));
break; break;
}
} }
}
}); });
}
}, },
componentWillUnmount () { componentWillUnmount () {
if (typeof this.subscription !== 'undefined') { if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe(); this.subscription.close();
this.subscription = null;
} }
}, },
...@@ -65,4 +70,4 @@ const PublicTimeline = React.createClass({ ...@@ -65,4 +70,4 @@ const PublicTimeline = React.createClass({
}); });
export default connect()(injectIntl(PublicTimeline)); export default connect(mapStateToProps)(injectIntl(PublicTimeline));
import WebSocketClient from 'websocket.js';
const createWebSocketURL = (url) => {
const a = document.createElement('a');
a.href = url;
a.href = a.href;
a.protocol = a.protocol.replace('http', 'ws');
return a.href;
};
export default function getStream(accessToken, stream, { connected, received, disconnected }) {
const ws = new WebSocketClient(`${createWebSocketURL(STREAMING_API_BASE_URL)}/api/v1/streaming/?access_token=${accessToken}&stream=${stream}`);
ws.onopen = connected;
ws.onmessage = e => received(JSON.parse(e.data));
ws.onclose = disconnected;
return ws;
};
- content_for :header_tags do - content_for :header_tags do
:javascript :javascript
window.STREAMING_API_BASE_URL = '#{Rails.configuration.x.streaming_api_base_url}';
window.INITIAL_STATE = #{json_escape(render(file: 'home/initial_state', formats: :json))} window.INITIAL_STATE = #{json_escape(render(file: 'home/initial_state', formats: :json))}
= javascript_include_tag 'application' = javascript_include_tag 'application'
......
...@@ -10,8 +10,10 @@ Rails.application.configure do ...@@ -10,8 +10,10 @@ Rails.application.configure do
config.x.use_s3 = ENV['S3_ENABLED'] == 'true' config.x.use_s3 = ENV['S3_ENABLED'] == 'true'
config.action_mailer.default_url_options = { host: host, protocol: https ? 'https://' : 'http://', trailing_slash: false } config.action_mailer.default_url_options = { host: host, protocol: https ? 'https://' : 'http://', trailing_slash: false }
config.x.streaming_api_base_url = 'http://localhost:4000'
if Rails.env.production? if Rails.env.production?
config.action_cable.allowed_request_origins = ["http#{https ? 's' : ''}://#{host}"] config.action_cable.allowed_request_origins = ["http#{https ? 's' : ''}://#{host}"]
config.x.streaming_api_base_url = ENV.fetch('STREAMING_API_BASE_URL') { "http#{https ? 's' : ''}://#{host}" }
end end
end end
...@@ -19,6 +19,16 @@ services: ...@@ -19,6 +19,16 @@ services:
volumes: volumes:
- ./public/assets:/mastodon/public/assets - ./public/assets:/mastodon/public/assets
- ./public/system:/mastodon/public/system - ./public/system:/mastodon/public/system
streaming:
restart: always
build: .
env_file: .env.production
command: npm run start
ports:
- "4000:4000"
depends_on:
- db
- redis
sidekiq: sidekiq:
restart: always restart: always
build: . build: .
......
...@@ -49,6 +49,22 @@ server { ...@@ -49,6 +49,22 @@ server {
tcp_nodelay on; tcp_nodelay on;
} }
location /api/v1/streaming {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_pass http://localhost:4000;
proxy_buffering off;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
tcp_nodelay on;
}
error_page 500 501 502 503 504 /500.html; error_page 500 501 502 503 504 /500.html;
} }
``` ```
...@@ -162,6 +178,27 @@ Restart=always ...@@ -162,6 +178,27 @@ Restart=always
WantedBy=multi-user.target WantedBy=multi-user.target
``` ```
Example systemd configuration file for the streaming API, to be placed in `/etc/systemd/system/mastodon-streaming.service`:
```systemd
[Unit]
Description=mastodon-streaming
After=network.target
[Service]
Type=simple
User=mastodon
WorkingDirectory=/home/mastodon/live
Environment="NODE_ENV=production"
Environment="PORT=4000"
ExecStart=/usr/bin/npm run start
TimeoutSec=15
Restart=always
[Install]
WantedBy=multi-user.target
```
This allows you to `sudo systemctl enable mastodon-*.service` and `sudo systemctl start mastodon-*.service` to get things going. This allows you to `sudo systemctl enable mastodon-*.service` and `sudo systemctl start mastodon-*.service` to get things going.
## Cronjobs ## Cronjobs
......
import dotenv from 'dotenv' import dotenv from 'dotenv'
import express from 'express' import express from 'express'
import http from 'http'
import redis from 'redis' import redis from 'redis'
import pg from 'pg' import pg from 'pg'
import log from 'npmlog' import log from 'npmlog'
import url from 'url'
import WebSocket from 'ws'
const env = process.env.NODE_ENV || 'development' const env = process.env.NODE_ENV || 'development'
...@@ -27,8 +30,10 @@ const pgConfigs = { ...@@ -27,8 +30,10 @@ const pgConfigs = {
} }
} }
const app = express() const app = express()
const pgPool = new pg.Pool(pgConfigs[env]) const pgPool = new pg.Pool(pgConfigs[env])
const server = http.createServer(app)
const wss = new WebSocket.Server({ server })
const allowCrossDomain = (req, res, next) => { const allowCrossDomain = (req, res, next) => {
res.header('Access-Control-Allow-Origin', '*') res.header('Access-Control-Allow-Origin', '*')
...@@ -38,22 +43,7 @@ const allowCrossDomain = (req, res, next) => { ...@@ -38,22 +43,7 @@ const allowCrossDomain = (req, res, next) => {
next() next()
} }
const authenticationMiddleware = (req, res, next) => { const accountFromToken = (token, req, next) => {
if (req.method === 'OPTIONS') {
return next()
}
const authorization = req.get('Authorization')
if (!authorization) {
const err = new Error('Missing access token')
err.statusCode = 401
return next(err)
}
const token = authorization.replace(/^Bearer /, '')
pgPool.connect((err, client, done) => { pgPool.connect((err, client, done) => {
if (err) { if (err) {
return next(err) return next(err)
...@@ -80,26 +70,36 @@ const authenticationMiddleware = (req, res, next) => { ...@@ -80,26 +70,36 @@ const authenticationMiddleware = (req, res, next) => {
}) })
} }
const authenticationMiddleware = (req, res, next) => {
if (req.method === 'OPTIONS') {
return next()
}
const authorization = req.get('Authorization')
if (!authorization) {
const err = new Error('Missing access token')
err.statusCode = 401
return next(err)
}
const token = authorization.replace(/^Bearer /, '')
accountFromToken(token, req, next)
}
const errorMiddleware = (err, req, res, next) => { const errorMiddleware = (err, req, res, next) => {
log.error(err) log.error(err)
res.writeHead(err.statusCode || 500, { 'Content-Type': 'application/json' }) res.writeHead(err.statusCode || 500, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ error: err.statusCode ? `${err}` : 'An unexpected error occured' })) res.end(JSON.stringify({ error: err.statusCode ? `${err}` : 'An unexpected error occurred' }))
} }
const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', '); const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', ');
const streamFrom = (id, req, res, needsFiltering = false) => { const streamFrom = (redisClient, id, req, output, needsFiltering = false) => {
log.verbose(`Starting stream from ${id} for ${req.accountId}`) log.verbose(`Starting stream from ${id} for ${req.accountId}`)
res.setHeader('Content-Type', 'text/event-stream')
res.setHeader('Transfer-Encoding', 'chunked')
const redisClient = redis.createClient({
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASSWORD
})
redisClient.on('message', (channel, message) => { redisClient.on('message', (channel, message) => {
const { event, payload } = JSON.parse(message) const { event, payload } = JSON.parse(message)
...@@ -127,36 +127,107 @@ const streamFrom = (id, req, res, needsFiltering = false) => { ...@@ -127,36 +127,107 @@ const streamFrom = (id, req, res, needsFiltering = false) => {
return return
} }
res.write(`event: ${event}\n`)