Commit ccb8ac85 authored by Eugen Rochko's avatar Eugen Rochko

Make the streaming API also handle websockets (because trying to get the...

Make the streaming API also handle websockets (because trying to get the browser EventSource interface to
work flawlessly was a nightmare). WARNING: This commit makes the web UI connect to the streaming API instead
of ActionCable like before. This means that if you are upgrading, you should set that up beforehand.
parent 8c0bc130
......@@ -43,5 +43,5 @@ SMTP_FROM_ADDRESS=notifications@example.com
# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front
# S3_CLOUDFRONT_HOST=
# Optional Firebase Cloud Messaging API key
FCM_API_KEY=
# Streaming API integration
# STREAMING_API_BASE_URL=
......@@ -13,4 +13,3 @@
//= require jquery
//= require jquery_ujs
//= require components
//= require cable
// Action Cable provides the framework to deal with WebSockets in Rails.
// You can generate new channels where WebSocket features live using the rails generate channel command.
//
//= require action_cable
//= require_self
(function() {
this.App || (this.App = {});
App.cable = ActionCable.createConsumer();
}).call(this);
......@@ -43,6 +43,7 @@ import hu from 'react-intl/locale-data/hu';
import uk from 'react-intl/locale-data/uk';
import getMessagesForLocale from '../locales';
import { hydrateStore } from '../actions/store';
import createStream from '../stream';
const store = configureStore();
......@@ -60,28 +61,27 @@ const Mastodon = React.createClass({
locale: React.PropTypes.string.isRequired
},
componentWillMount() {
const { locale } = this.props;
if (typeof App !== 'undefined') {
this.subscription = App.cable.subscriptions.create('TimelineChannel', {
received (data) {
switch(data.event) {
case 'update':
store.dispatch(updateTimeline('home', JSON.parse(data.payload)));
break;
case 'delete':
store.dispatch(deleteFromTimelines(data.payload));
break;
case 'notification':
store.dispatch(updateNotifications(JSON.parse(data.payload), getMessagesForLocale(locale), locale));
break;
}
componentDidMount() {
const { locale } = this.props;
const accessToken = store.getState().getIn(['meta', 'access_token']);
this.subscription = createStream(accessToken, 'user', {
received (data) {
switch(data.event) {
case 'update':
store.dispatch(updateTimeline('home', JSON.parse(data.payload)));
break;
case 'delete':
store.dispatch(deleteFromTimelines(data.payload));
break;
case 'notification':
store.dispatch(updateNotifications(JSON.parse(data.payload), getMessagesForLocale(locale), locale));
break;
}
}
});
}
});
// Desktop notifications
if (typeof window.Notification !== 'undefined' && Notification.permission === 'default') {
......@@ -91,7 +91,8 @@ const Mastodon = React.createClass({
componentWillUnmount () {
if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe();
this.subscription.close();
this.subscription = null;
}
},
......
......@@ -8,45 +8,49 @@ import {
deleteFromTimelines
} from '../../actions/timelines';
import ColumnBackButtonSlim from '../../components/column_back_button_slim';
import createStream from '../../stream';
const mapStateToProps = state => ({
accessToken: state.getIn(['meta', 'access_token'])
});
const HashtagTimeline = React.createClass({
propTypes: {
params: React.PropTypes.object.isRequired,
dispatch: React.PropTypes.func.isRequired
dispatch: React.PropTypes.func.isRequired,
accessToken: React.PropTypes.string.isRequired
},
mixins: [PureRenderMixin],
_subscribe (dispatch, id) {
if (typeof App !== 'undefined') {
this.subscription = App.cable.subscriptions.create({
channel: 'HashtagChannel',
tag: id
}, {
received (data) {
switch(data.event) {
case 'update':
dispatch(updateTimeline('tag', JSON.parse(data.payload)));
break;
case 'delete':
dispatch(deleteFromTimelines(data.payload));
break;
}
const { accessToken } = this.props;
this.subscription = createStream(accessToken, `hashtag&tag=${id}`, {
received (data) {
switch(data.event) {
case 'update':
dispatch(updateTimeline('tag', JSON.parse(data.payload)));
break;
case 'delete':
dispatch(deleteFromTimelines(data.payload));
break;
}
}
});
}
});
},
_unsubscribe () {
if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe();
this.subscription.close();
this.subscription = null;
}
},
componentWillMount () {
componentDidMount () {
const { dispatch } = this.props;
const { id } = this.props.params;
......@@ -79,4 +83,4 @@ const HashtagTimeline = React.createClass({
});
export default connect()(HashtagTimeline);
export default connect(mapStateToProps)(HashtagTimeline);
......@@ -9,46 +9,51 @@ import {
} from '../../actions/timelines';
import { defineMessages, injectIntl } from 'react-intl';
import ColumnBackButtonSlim from '../../components/column_back_button_slim';
import createStream from '../../stream';
const messages = defineMessages({
title: { id: 'column.public', defaultMessage: 'Public' }
});
const mapStateToProps = state => ({
accessToken: state.getIn(['meta', 'access_token'])
});
const PublicTimeline = React.createClass({
propTypes: {
dispatch: React.PropTypes.func.isRequired,
intl: React.PropTypes.object.isRequired
intl: React.PropTypes.object.isRequired,
accessToken: React.PropTypes.string.isRequired
},
mixins: [PureRenderMixin],
componentWillMount () {
const { dispatch } = this.props;
componentDidMount () {
const { dispatch, accessToken } = this.props;
dispatch(refreshTimeline('public'));
if (typeof App !== 'undefined') {
this.subscription = App.cable.subscriptions.create('PublicChannel', {
this.subscription = createStream(accessToken, 'public', {
received (data) {
switch(data.event) {
case 'update':
dispatch(updateTimeline('public', JSON.parse(data.payload)));
break;
case 'delete':
dispatch(deleteFromTimelines(data.payload));
break;
}
received (data) {
switch(data.event) {
case 'update':
dispatch(updateTimeline('public', JSON.parse(data.payload)));
break;
case 'delete':
dispatch(deleteFromTimelines(data.payload));
break;
}
}
});
}
});
},
componentWillUnmount () {
if (typeof this.subscription !== 'undefined') {
this.subscription.unsubscribe();
this.subscription.close();
this.subscription = null;
}
},
......@@ -65,4 +70,4 @@ const PublicTimeline = React.createClass({
});
export default connect()(injectIntl(PublicTimeline));
export default connect(mapStateToProps)(injectIntl(PublicTimeline));
import WebSocketClient from 'websocket.js';
const createWebSocketURL = (url) => {
const a = document.createElement('a');
a.href = url;
a.href = a.href;
a.protocol = a.protocol.replace('http', 'ws');
return a.href;
};
export default function getStream(accessToken, stream, { connected, received, disconnected }) {
const ws = new WebSocketClient(`${createWebSocketURL(STREAMING_API_BASE_URL)}/api/v1/streaming/?access_token=${accessToken}&stream=${stream}`);
ws.onopen = connected;
ws.onmessage = e => received(JSON.parse(e.data));
ws.onclose = disconnected;
return ws;
};
- content_for :header_tags do
:javascript
window.STREAMING_API_BASE_URL = '#{Rails.configuration.x.streaming_api_base_url}';
window.INITIAL_STATE = #{json_escape(render(file: 'home/initial_state', formats: :json))}
= javascript_include_tag 'application'
......
......@@ -10,8 +10,10 @@ Rails.application.configure do
config.x.use_s3 = ENV['S3_ENABLED'] == 'true'
config.action_mailer.default_url_options = { host: host, protocol: https ? 'https://' : 'http://', trailing_slash: false }
config.x.streaming_api_base_url = 'http://localhost:4000'
if Rails.env.production?
config.action_cable.allowed_request_origins = ["http#{https ? 's' : ''}://#{host}"]
config.x.streaming_api_base_url = ENV.fetch('STREAMING_API_BASE_URL') { "http#{https ? 's' : ''}://#{host}" }
end
end
......@@ -19,6 +19,16 @@ services:
volumes:
- ./public/assets:/mastodon/public/assets
- ./public/system:/mastodon/public/system
streaming:
restart: always
build: .
env_file: .env.production
command: npm run start
ports:
- "4000:4000"
depends_on:
- db
- redis
sidekiq:
restart: always
build: .
......
......@@ -49,6 +49,22 @@ server {
tcp_nodelay on;
}
location /api/v1/streaming {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_pass http://localhost:4000;
proxy_buffering off;
proxy_redirect off;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
tcp_nodelay on;
}
error_page 500 501 502 503 504 /500.html;
}
```
......@@ -162,6 +178,27 @@ Restart=always
WantedBy=multi-user.target
```
Example systemd configuration file for the streaming API, to be placed in `/etc/systemd/system/mastodon-streaming.service`:
```systemd
[Unit]
Description=mastodon-streaming
After=network.target
[Service]
Type=simple
User=mastodon
WorkingDirectory=/home/mastodon/live
Environment="NODE_ENV=production"
Environment="PORT=4000"
ExecStart=/usr/bin/npm run start
TimeoutSec=15
Restart=always
[Install]
WantedBy=multi-user.target
```
This allows you to `sudo systemctl enable mastodon-*.service` and `sudo systemctl start mastodon-*.service` to get things going.
## Cronjobs
......
import dotenv from 'dotenv'
import express from 'express'
import http from 'http'
import redis from 'redis'
import pg from 'pg'
import log from 'npmlog'
import url from 'url'
import WebSocket from 'ws'
const env = process.env.NODE_ENV || 'development'
......@@ -27,8 +30,10 @@ const pgConfigs = {
}
}
const app = express()
const app = express()
const pgPool = new pg.Pool(pgConfigs[env])
const server = http.createServer(app)
const wss = new WebSocket.Server({ server })
const allowCrossDomain = (req, res, next) => {
res.header('Access-Control-Allow-Origin', '*')
......@@ -38,22 +43,7 @@ const allowCrossDomain = (req, res, next) => {
next()
}
const authenticationMiddleware = (req, res, next) => {
if (req.method === 'OPTIONS') {
return next()
}
const authorization = req.get('Authorization')
if (!authorization) {
const err = new Error('Missing access token')
err.statusCode = 401
return next(err)
}
const token = authorization.replace(/^Bearer /, '')
const accountFromToken = (token, req, next) => {
pgPool.connect((err, client, done) => {
if (err) {
return next(err)
......@@ -80,26 +70,36 @@ const authenticationMiddleware = (req, res, next) => {
})
}
const authenticationMiddleware = (req, res, next) => {
if (req.method === 'OPTIONS') {
return next()
}
const authorization = req.get('Authorization')
if (!authorization) {
const err = new Error('Missing access token')
err.statusCode = 401
return next(err)
}
const token = authorization.replace(/^Bearer /, '')
accountFromToken(token, req, next)
}
const errorMiddleware = (err, req, res, next) => {
log.error(err)
res.writeHead(err.statusCode || 500, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ error: err.statusCode ? `${err}` : 'An unexpected error occured' }))
res.end(JSON.stringify({ error: err.statusCode ? `${err}` : 'An unexpected error occurred' }))
}
const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', ');
const streamFrom = (id, req, res, needsFiltering = false) => {
const streamFrom = (redisClient, id, req, output, needsFiltering = false) => {
log.verbose(`Starting stream from ${id} for ${req.accountId}`)
res.setHeader('Content-Type', 'text/event-stream')
res.setHeader('Transfer-Encoding', 'chunked')
const redisClient = redis.createClient({
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASSWORD
})
redisClient.on('message', (channel, message) => {
const { event, payload } = JSON.parse(message)
......@@ -127,36 +127,107 @@ const streamFrom = (id, req, res, needsFiltering = false) => {
return
}
res.write(`event: ${event}\n`)
res.write(`data: ${payload}\n\n`)
log.silly(`Transmitting for ${req.accountId}: ${event} ${payload}`)
output(event, payload)
})
})
} else {
res.write(`event: ${event}\n`)
res.write(`data: ${payload}\n\n`)
log.silly(`Transmitting for ${req.accountId}: ${event} ${payload}`)
output(event, payload)
}
})
redisClient.subscribe(id)
}
// Setup stream output to HTTP
const streamToHttp = (req, res, redisClient) => {
res.setHeader('Content-Type', 'text/event-stream')
res.setHeader('Transfer-Encoding', 'chunked')
const heartbeat = setInterval(() => res.write(':thump\n'), 15000)
req.on('close', () => {
log.verbose(`Ending stream from ${id} for ${req.accountId}`)
log.verbose(`Ending stream for ${req.accountId}`)
clearInterval(heartbeat)
redisClient.quit()
})
redisClient.subscribe(id)
return (event, payload) => {
res.write(`event: ${event}\n`)
res.write(`data: ${payload}\n\n`)
}
}
// Setup stream output to WebSockets
const streamToWs = (req, ws, redisClient) => {
ws.on('close', () => {
log.verbose(`Ending stream for ${req.accountId}`)
redisClient.quit()
})
return (event, payload) => {
ws.send(JSON.stringify({ event, payload }))
}
}
// Get new redis connection
const getRedisClient = () => redis.createClient({
host: process.env.REDIS_HOST || '127.0.0.1',
port: process.env.REDIS_PORT || 6379,
password: process.env.REDIS_PASSWORD
})
app.use(allowCrossDomain)
app.use(authenticationMiddleware)
app.use(errorMiddleware)
app.get('/api/v1/streaming/user', (req, res) => streamFrom(`timeline:${req.accountId}`, req, res))
app.get('/api/v1/streaming/public', (req, res) => streamFrom('timeline:public', req, res, true))
app.get('/api/v1/streaming/hashtag', (req, res) => streamFrom(`timeline:hashtag:${req.params.tag}`, req, res, true))
app.get('/api/v1/streaming/user', (req, res) => {
const redisClient = getRedisClient()
streamFrom(redisClient, `timeline:${req.accountId}`, req, streamToHttp(req, res, redisClient))
})
app.get('/api/v1/streaming/public', (req, res) => {
const redisClient = getRedisClient()
streamFrom(redisClient, 'timeline:public', req, streamToHttp(req, res, redisClient), true)
})
app.get('/api/v1/streaming/hashtag', (req, res) => {
const redisClient = getRedisClient()
streamFrom(redisClient, `timeline:hashtag:${req.params.tag}`, req, streamToHttp(req, res, redisClient), true)
})
log.level = 'verbose'
log.info(`Starting HTTP server on port ${process.env.PORT || 4000}`)
wss.on('connection', ws => {
const location = url.parse(ws.upgradeReq.url, true)
const token = location.query.access_token
const req = {}
app.listen(process.env.PORT || 4000)
accountFromToken(token, req, err => {
if (err) {
log.error(err)
ws.close()
return
}
const redisClient = getRedisClient()
switch(location.query.stream) {
case 'user':
streamFrom(redisClient, `timeline:${req.accountId}`, req, streamToWs(req, ws, redisClient))
break;
case 'public':
streamFrom(redisClient, 'timeline:public', req, streamToWs(req, ws, redisClient), true)
break;
case 'hashtag':
streamFrom(redisClient, `timeline:hashtag:${location.query.tag}`, req, streamToWs(req, ws, redisClient), true)
break;
default:
ws.close()
}
})
})
server.listen(process.env.PORT || 4000, () => {
log.level = process.env.LOG_LEVEL || 'verbose'
log.info(`Starting streaming API server on port ${server.address().port}`)
})
......@@ -1237,6 +1237,12 @@ babylon@^6.15.0:
version "6.15.0"
resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.15.0.tgz#ba65cfa1a80e1759b0e89fb562e27dccae70348e"
backoff@^2.4.1:
version "2.5.0"
resolved "https://registry.yarnpkg.com/backoff/-/backoff-2.5.0.tgz#f616eda9d3e4b66b8ca7fca79f695722c5f8e26f"
dependencies: