From 108d7ad0b764a9e5432374b7a9b44c66f61190d5 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sun, 23 Aug 2020 14:41:48 +0000 Subject: [PATCH 01/31] Fix /api/instance/config --- src/http.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http.ts b/src/http.ts index ef4e127..fe02159 100644 --- a/src/http.ts +++ b/src/http.ts @@ -156,7 +156,7 @@ async function initAPI() { ping_timeout: config['rtmp']['ping_timeout'] }, media: { - vods: config['config']['media']['record'], + vods: config['media']['record'], publicEndpoint: config['media']['publicEndpoint'], privateEndpoint: config['media']['privateEndpoint'], adaptive: config['transcode']['adaptive'] From 15824d25c07e497487e37f91728e7defd387b6ba Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Mon, 24 Aug 2020 18:14:26 +0200 Subject: [PATCH 02/31] Add live field to /api/users/all --- docs/REST.md | 4 +++- src/http.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/REST.md b/docs/REST.md index 3f52bb7..2832a7e 100644 --- a/docs/REST.md +++ b/docs/REST.md @@ -77,7 +77,9 @@ The array will be wrapped in a JSON object under the key 'users'. ## /api/users/all -Same as above, but returns all users regardless of whether they are streaming. Also unfinished. +Same as above, but returns all users regardless of whether they are streaming and if they're streaming or not. Also unfinished. + +**Example**: `{users: [{username:"foo", title:"bar", live:1}] }` diff --git a/src/http.ts b/src/http.ts index fe02159..7474e5d 100644 --- a/src/http.ts +++ b/src/http.ts @@ -194,7 +194,7 @@ async function initAPI() { }); }); app.post('/api/users/all', (req, res) => { - let qs = 'SELECT username,title FROM user_meta'; + let qs = 'SELECT username,title,live FROM user_meta'; if(req.body.sort) { switch (req.body.sort) { From 2d7bc8204f59526deb0746af6586b7d2de7a0cf4 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 31 Aug 2020 01:26:43 +0000 Subject: [PATCH 03/31] Update REST.md --- docs/REST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/REST.md b/docs/REST.md index 2832a7e..74786c9 100644 --- a/docs/REST.md +++ b/docs/REST.md @@ -77,7 +77,7 @@ The array will be wrapped in a JSON object under the key 'users'. ## /api/users/all -Same as above, but returns all users regardless of whether they are streaming and if they're streaming or not. Also unfinished. +Same as above, but returns all users regardless of whether they are streaming and if they're streaming or not. **Example**: `{users: [{username:"foo", title:"bar", live:1}] }` From ee3527f2928ec27474c77d7e2ea41f56c54f50d8 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 10 Oct 2020 15:55:32 -0500 Subject: [PATCH 04/31] Implement database versioning and migration. It could not possibly be any simpler or easier to break, but it works. And it can be used to automatically migrate to a better system for migration later. For now, the way it works is by creating a new migration script with the name of the version (increment by one, whole numbers) in the src/db folder On start up, it will compare version numbers and run new scripts. The user can also manually check for migrations and skip the automatic checking. Added a bit of additional logging to see what's happening in the startup process as well. --- README.md | 8 +++++++- package.json | 3 ++- src/cleanup.ts | 34 +++++++++++++++++++++++++++++++++- src/config.ts | 1 + src/database.ts | 1 + src/db/0.ts | 8 ++++++++ src/index.ts | 2 +- src/migrate.ts | 9 +++++++++ 8 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/db/0.ts create mode 100644 src/migrate.ts diff --git a/README.md b/README.md index cddcc6a..2018270 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,13 @@ Follow the instructions after setup runs. ### Run the server ```bash -npm start +npm run start +``` +You can also run this to skip checking the database version on startup. +```bash +npm run start -- --skip-migrate +# don't forget to migrate manually when you update +npm run migrate ``` ## Contributing diff --git a/package.json b/package.json index f77afd1..840082b 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "scripts": { "start": "ts-node src/index.ts", "user": "ts-node src/cli.ts", - "setup": "sh install/setup.sh" + "setup": "sh install/setup.sh", + "migrate": "ts-node src/migrate.ts" }, "repository": { "type": "git", diff --git a/src/cleanup.ts b/src/cleanup.ts index 00e55ae..56c9dc9 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -1,10 +1,42 @@ import * as db from "./database"; +import {readdirSync} from "fs"; -async function init() { +async function init(m?: boolean) { + if(!m){ + console.log('Checking database version.'); + var tmp: string[] = await db.query('show tables like \"db_meta\"'); + if(tmp.length === 0){ + console.log('No database version info, running initial migration.'); + await require('./db/0').run(); + await bringUpToDate(); + } + else { + await bringUpToDate(); + } + } + else { + console.log('Skipping database version check.'); + } //If satyr is restarted in the middle of a stream //it causes problems //Live flags in the database stay live await db.query('update user_meta set live=false'); } +async function bringUpToDate(): Promise{ + var versions: Object[] = await db.query('select * from db_meta'); + var scripts: Buffer[] | string[] = readdirSync('./src/db/', {withFileTypes: false}); + var diff: number = scripts.length - versions.length + if(diff === 0){ + console.log('No migration needed.'); + } else { + console.log('Versions differ, migrating now.'); + for(let i=0;i {process.exit()}); \ No newline at end of file From 7b25a7bc970f1ed7322f655180cc3a46e9f95308 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 10 Oct 2020 16:14:53 -0500 Subject: [PATCH 05/31] Increment minor version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 840082b..f9aae24 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.9.2", + "version": "0.9.3", "description": "A livestreaming server.", "license": "AGPL-3.0", "author": "knotteye", From 5fe4728c1103f7ba51a07536a751cd9793962e41 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 10 Oct 2020 16:35:05 -0500 Subject: [PATCH 06/31] Add migration script and update remove and adduser functions. Needs a UI, API, and functionality. --- src/database.ts | 3 +++ src/db/1.ts | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 src/db/1.ts diff --git a/src/database.ts b/src/database.ts index 2cbc440..1ee8a17 100644 --- a/src/database.ts +++ b/src/database.ts @@ -22,6 +22,7 @@ async function addUser(name: string, password: string){ await query('INSERT INTO users (username, password_hash, stream_key, record_flag) VALUES ('+raw.escape(name)+', '+raw.escape(hash)+', '+raw.escape(key)+', 0)'); await query('INSERT INTO user_meta (username, title, about, live) VALUES ('+raw.escape(name)+',\'\',\'\',false)'); await query('INSERT INTO chat_integration (username, irc, xmpp, twitch, discord) VALUES ('+raw.escape(name)+',\'\',\'\',\'\',\'\')'); + await query('INSERT INTO twitch_mirror (username) VALUES ('+raw.escape(name)+')'); return true; } @@ -30,6 +31,8 @@ async function rmUser(name: string){ if(!exist[0]) return false; await query('delete from users where username='+raw.escape(name)+' limit 1'); await query('delete from user_meta where username='+raw.escape(name)+' limit 1'); + await query('delete from chat_integration where username='+raw.escape(name)+' limit 1'); + await query('delete from twitch_mirror where username='+raw.escape(name)+' limit 1'); return true; } diff --git a/src/db/1.ts b/src/db/1.ts new file mode 100644 index 0000000..55b8d89 --- /dev/null +++ b/src/db/1.ts @@ -0,0 +1,9 @@ +import * as db from "../database"; + +async function run () { + await db.query('CREATE TABLE IF NOT EXISTS twitch_mirror(username VARCHAR(25), enabled TINYINT DEFAULT 0, twitch_key VARCHAR(50) DEFAULT \"\")'); + await db.query('INSERT INTO twitch_mirror(username) SELECT username FROM users'); + await db.query('INSERT INTO db_meta (version) VALUES (1)'); +} + +export { run } \ No newline at end of file From 4ff4a6329dfc924a4b9974957233d5e5139b8345 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 10:54:55 -0500 Subject: [PATCH 07/31] Add configuration options for twitch mirror --- install/config.example.yml | 6 +++++- src/config.ts | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/install/config.example.yml b/install/config.example.yml index da27427..fe66753 100644 --- a/install/config.example.yml +++ b/install/config.example.yml @@ -56,4 +56,8 @@ chat: enabled: false username: #https://twitchapps.com/tmi/ - password: \ No newline at end of file + password: + +twitch_mirror: +# enable to allow users to mirror video streams to twitch + enabled: false \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 3f74000..d118a25 100644 --- a/src/config.ts +++ b/src/config.ts @@ -81,6 +81,9 @@ const config: Object = { username: null, token: null }, localconfig['chat']['twitch']) - } + }, + twitch_mirror: Object.assign({ + enabled: false + }, localconfig['twitch_mirror']) }; export { config }; \ No newline at end of file From 44cc3213ca780bf274818b69c4b1999edbe8d165 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 11:14:59 -0500 Subject: [PATCH 08/31] Tweak config changes, add functionality in server.ts Still needs an API and a UI, then good to go. --- install/config.example.yml | 6 +++++- src/config.ts | 3 ++- src/server.ts | 9 +++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/install/config.example.yml b/install/config.example.yml index fe66753..c67d16b 100644 --- a/install/config.example.yml +++ b/install/config.example.yml @@ -60,4 +60,8 @@ chat: twitch_mirror: # enable to allow users to mirror video streams to twitch - enabled: false \ No newline at end of file +# for those with truly no bandwidth limits + enabled: false + # https://stream.twitch.tv/ingests/ + # do not include {stream_key} + ingest: 'rtmp://live-ord02.twitch.tv/app/ \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index d118a25..4bd6ec0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -83,7 +83,8 @@ const config: Object = { }, localconfig['chat']['twitch']) }, twitch_mirror: Object.assign({ - enabled: false + enabled: false, + ingest: null }, localconfig['twitch_mirror']) }; export { config }; \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index dd58ee5..fcd409d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -68,6 +68,15 @@ function init () { console.log('[NodeMediaServer] Skipping recording for stream:',id); } db.query('update user_meta set live=true where username=\''+results[0].username+'\' limit 1'); + db.query('SELECT twitch_key,enabled from twitch_mirror where username='+db.raw.escape(results[0].username)+' limit 1').then(async (tm) => { + if(!tm[0]['enabled'] || !config['twitch_mirror']['enabled'] || !config['twitch_mirror']['ingest']) return; + else + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'libx264', '-acodec', 'libaac', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { + detached: true, + stdio : 'inherit', + maxBuffer: Infinity + }).unref(); + }); console.log('[NodeMediaServer] Stream key ok for stream:',id); } else{ From 98927bd7b84ddba895f341670e09a661f28bbfb2 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 12:11:04 -0500 Subject: [PATCH 09/31] Add API functionality for twitch mirror. --- src/api.ts | 15 +++++++++++++-- src/http.ts | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/api.ts b/src/api.ts index bcc79ae..a803d63 100644 --- a/src/api.ts +++ b/src/api.ts @@ -18,7 +18,7 @@ async function register(name: string, password: string, confirm: string): Promis } async function update(fields: object): Promise{ - if(!fields['title'] && !fields['bio'] && (fields['rec'] !== 'true' && fields['rec'] !== 'false')) return {"error":"no valid fields specified"}; + if(!fields['title'] && !fields['bio'] && (fields['rec'] !== 'true' && fields['rec'] !== 'false') && (fields['twitch'] !== 'true' && fields['twitch'] !== 'false') && !fields['twitch_key']) return {"error":"no valid fields specified"}; let qs: string = ""; let f: boolean = false; if(fields['title']) {qs += ' user_meta.title='+db.raw.escape(fields['title']);f = true;} @@ -30,8 +30,19 @@ async function update(fields: object): Promise{ if(typeof(fields['rec']) === 'boolean' || typeof(fields['rec']) === 'number') { if(f) qs+=','; qs += ' users.record_flag='+db.raw.escape(fields['rec']); + f=true; + } + if(typeof(fields['twitch']) === 'boolean' || typeof(fields['twitch']) === 'number') { + if(f) qs+=','; + qs += ' twitch_mirror.enabled='+db.raw.escape(fields['twitch']); + f=true; + } + if(fields['twitch_key']){ + if(f) qs+=','; + qs += ' twitch_mirror.twitch_key='+db.raw.escape(fields['twitch_key']); + f = true; } - await db.query('UPDATE users,user_meta SET'+qs+' WHERE users.username='+db.raw.escape(fields['name'])+' AND user_meta.username='+db.raw.escape(fields['name'])); + await db.query('UPDATE users,user_meta,twitch_mirror SET'+qs+' WHERE users.username='+db.raw.escape(fields['name'])+' AND user_meta.username='+db.raw.escape(fields['name'])+' AND twitch_mirror.username='+db.raw.escape(fields['name'])); return {success:""}; } diff --git a/src/http.ts b/src/http.ts index 7474e5d..a179b2d 100644 --- a/src/http.ts +++ b/src/http.ts @@ -238,10 +238,14 @@ async function initAPI() { if(t) { if(req.body.record === "true") req.body.record = true; else if(req.body.record === "false") req.body.record = false; + if(req.body.twitch === "true") req.body.twitch = true; + else if(req.body.twitch === "false") req.body.twitch = false; return api.update({name: t['username'], title: "title" in req.body ? req.body.title : false, bio: "bio" in req.body ? req.body.bio : false, - rec: "record" in req.body ? req.body.record : "NA" + rec: "record" in req.body ? req.body.record : "NA", + twitch: "twitch" in req.body ? req.body.twitch: "NA", + twitch_key: "twitch_key" in req.body ? req.body.twitch_key : false }).then((r) => { res.json(r); return; From d4bb2ceebec8bd39c7ee3da14a40c4d98b4427c3 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 12:12:27 -0500 Subject: [PATCH 10/31] Update documentation for API. All that's left for twitch mirroring is a UI and then testing. --- docs/REST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/REST.md b/docs/REST.md index 74786c9..3160cf7 100644 --- a/docs/REST.md +++ b/docs/REST.md @@ -124,9 +124,9 @@ Update the current user's information **Authentication**: yes -**Parameters**: title, bio, rec +**Parameters**: title, bio, rec, twitch, twitch_key -Rec is a boolean (whether to record VODs), others are strings. Parameters that are not included in the request will not be updated. +Rec is a boolean (whether to record VODs), twitch is a boolean (whether to mirror video streams to twitch) others are strings. Twitch_key is the stream key to use for twitch. Parameters that are not included in the request will not be updated. **Response**: Returns `{error: "error code"}` or `{success: ""}` From 93738d27bc15a3daa32cc88fd217eeb3eb383d5c Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 13:34:24 -0500 Subject: [PATCH 11/31] Add sections in profile.njk for adjusting settings. Everything tested and working apart from the actual streaming functionality. --- src/http.ts | 4 +++- templates/profile.njk | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/http.ts b/src/http.ts index a179b2d..b5f9c53 100644 --- a/src/http.ts +++ b/src/http.ts @@ -496,7 +496,9 @@ async function initSite(openReg) { if(tryDecode(req.cookies.Authorization)) { db.query('select * from user_meta where username='+db.raw.escape(JWT.decode(req.cookies.Authorization)['username'])).then((result) => { db.query('select record_flag from users where username='+db.raw.escape(JWT.decode(req.cookies.Authorization)['username'])).then((r2) => { - res.render('profile.njk', Object.assign({rflag: r2[0]}, {meta: result[0]}, {auth: {is: true, name: JWT.decode(req.cookies.Authorization)['username']}}, njkconf)); + db.query('select enabled from twitch_mirror where username='+db.raw.escape(JWT.decode(req.cookies.Authorization)['username'])).then((r3) => { + res.render('profile.njk', Object.assign({twitch: r3[0]}, {rflag: r2[0]}, {meta: result[0]}, {auth: {is: true, name: JWT.decode(req.cookies.Authorization)['username']}}, njkconf)); + }); }); }); //res.render('profile.njk', Object.assign({auth: {is: true, name: JWT.decode(req.cookies.Authorization)['username']}}, njkconf)); diff --git a/templates/profile.njk b/templates/profile.njk index 465a8da..33ed1dd 100644 --- a/templates/profile.njk +++ b/templates/profile.njk @@ -5,7 +5,9 @@
Stream Title:

Bio:

- Record VODs: Yes No

+ ReStream to Twitch: Yes No
+ Record VODs: Yes No
+ Twitch Key:


From 7b84253fc145ae9743cf0a782bf4bcbdac921cd5 Mon Sep 17 00:00:00 2001 From: knotteye Date: Mon, 12 Oct 2020 20:53:22 -0500 Subject: [PATCH 12/31] Add some logging for twitch mirror --- src/server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.ts b/src/server.ts index fcd409d..dd60538 100644 --- a/src/server.ts +++ b/src/server.ts @@ -70,8 +70,8 @@ function init () { db.query('update user_meta set live=true where username=\''+results[0].username+'\' limit 1'); db.query('SELECT twitch_key,enabled from twitch_mirror where username='+db.raw.escape(results[0].username)+' limit 1').then(async (tm) => { if(!tm[0]['enabled'] || !config['twitch_mirror']['enabled'] || !config['twitch_mirror']['ingest']) return; - else - execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'libx264', '-acodec', 'libaac', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { + console.log('[NodeMediaServer] Mirroring to twitch for stream:',id) + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { detached: true, stdio : 'inherit', maxBuffer: Infinity From 987d837ee6b592ac4eb4dc6085e4331c2bc78885 Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 15:07:30 -0500 Subject: [PATCH 13/31] Update some dependency versions. --- package-lock.json | 168 +++++++++++++++++++++++----------------------- package.json | 6 +- 2 files changed, 88 insertions(+), 86 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7428239..78361b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.7.2", + "version": "0.9.3", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -18,9 +18,9 @@ } }, "@types/node": { - "version": "12.7.5", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.7.5.tgz", - "integrity": "sha512-9fq4jZVhPNW8r+UYKnxF1e2HkDWOWKM5bC2/7c9wPV835I0aOrVbS/Hw/pWPk2uKrNXQqg9Z959Kz+IYDd5p3w==" + "version": "12.12.67", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.67.tgz", + "integrity": "sha512-R48tgL2izApf+9rYNH+3RBMbRpPeW3N8f0I9HMhggeq4UXwBDqumJ14SDs4ctTMhG11pIOduZ4z3QWGOiMc9Vg==" }, "a-sync-waterfall": { "version": "1.0.1", @@ -149,12 +149,12 @@ "integrity": "sha1-/bC0OWLKe0BFanwrtI/hc9otISI=" }, "bcrypt": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-3.0.6.tgz", - "integrity": "sha512-taA5bCTfXe7FUjKroKky9EXpdhkVvhE5owfxfLYodbrAR1Ul3juLmIQmIQBK4L9a5BuUcE6cqmwT+Da20lF9tg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-5.0.0.tgz", + "integrity": "sha512-jB0yCBl4W/kVHM2whjfyqnxTmOHkCX4kHEa5nYKSoGeYe8YrjTYTc87/6bwt1g8cmV0QrbhKriETg9jWtcREhg==", "requires": { - "nan": "2.13.2", - "node-pre-gyp": "0.12.0" + "node-addon-api": "^3.0.0", + "node-pre-gyp": "0.15.0" } }, "better-assert": { @@ -278,9 +278,9 @@ } }, "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==" + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, "code-point-at": { "version": "1.1.0", @@ -773,9 +773,9 @@ } }, "glob": { - "version": "7.1.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", - "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -861,9 +861,9 @@ } }, "ignore-walk": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.2.tgz", - "integrity": "sha512-EXyErtpHbn75ZTsOADsfx6J/FPo6/5cjev46PXrcTpd8z3BoRkXgYu9/JVqrI7tusjmwCZutGeRJeU0Wo1e4Cw==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", "requires": { "minimatch": "^3.0.4" } @@ -964,9 +964,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.camelcase": { "version": "4.3.0", @@ -1042,21 +1042,26 @@ "brace-expansion": "^1.1.7" } }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, "minipass": { - "version": "2.6.5", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.6.5.tgz", - "integrity": "sha512-ewSKOPFH9blOLXx0YSE+mbrNMBFPS+11a2b03QZ+P4LVrUHW/GAlqeYC7DBknDyMWkHzrzTpDhUvy7MUxqyrPA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" } }, "minizlib": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.2.tgz", - "integrity": "sha512-hR3At21uSrsjjDTWrbu0IMLTpnkpv8IIMFDFaoz43Tmu4LkmAXfH44vNNzpTnf+OAQQCHrb91y/wc2J4x5XgSQ==", + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", "requires": { - "minipass": "^2.2.1" + "minipass": "^2.9.0" } }, "mkdirp": { @@ -1065,13 +1070,6 @@ "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "requires": { "minimist": "^1.2.5" - }, - "dependencies": { - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" - } } }, "moment": { @@ -1098,12 +1096,13 @@ "nan": { "version": "2.13.2", "resolved": "https://registry.npmjs.org/nan/-/nan-2.13.2.tgz", - "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==" + "integrity": "sha512-TghvYc72wlMGMVMluVo9WRJc0mB8KxxF/gZ4YYFy7V2ZQX9l7rgbPg7vjS9mt6U5HXODVFVI2bOduCzwOMv/lw==", + "optional": true }, "needle": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/needle/-/needle-2.4.0.tgz", - "integrity": "sha512-4Hnwzr3mi5L97hMYeNl8wRW/Onhy4nUKR/lVemJ8gJedxxUyBLm9kkrDColJvoSfwi0jCNhD+xCdOtiGDQiRZg==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.5.2.tgz", + "integrity": "sha512-LbRIwS9BfkPvNwNHlsA41Q29kL2L/6VaOJ0qisM5lLWsTV3nP15abO5ITL6L81zqFhzjRKDAYjpcBcwM0AVvLQ==", "requires": { "debug": "^3.2.6", "iconv-lite": "^0.4.4", @@ -1115,6 +1114,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "node-addon-api": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.0.2.tgz", + "integrity": "sha512-+D4s2HCnxPd5PjjI0STKwncjXTUKKqm74MDMz9OPXavjsGmjkvwgLtA5yoxJUdmpj52+2u+RrXgPipahKczMKg==" + }, "node-icu-charset-detector": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/node-icu-charset-detector/-/node-icu-charset-detector-0.2.0.tgz", @@ -1125,40 +1129,47 @@ } }, "node-media-server": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/node-media-server/-/node-media-server-2.1.3.tgz", - "integrity": "sha512-BZf39fpVDSVQT2E+8DqSVOb7oo31rcbA36l9sqtSuyZhBdxjidL5Nk2/G/2vqMGR9Q4JKzkTskGay2dWy5ZsUQ==", + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/node-media-server/-/node-media-server-2.2.4.tgz", + "integrity": "sha512-2Y5hZ+BI2YxM5+PiEXM9isAZUPSJoENTb0xXVzg8MzP9nFtVVv+X7+iGnFeyXB0BWaCsdBFD5A/rTL4dfaCw+Q==", "requires": { "basic-auth-connect": "^1.0.0", "chalk": "^2.4.2", "dateformat": "^3.0.3", "express": "^4.16.4", "lodash": ">=4.17.13", - "mkdirp": "^0.5.1", + "mkdirp": "1.0.3", "ws": "^5.2.2" + }, + "dependencies": { + "mkdirp": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.3.tgz", + "integrity": "sha512-6uCP4Qc0sWsgMLy1EOqqS/3rjDHOEnsStVr/4vtAIK2Y5i2kA7lFFejYrpIyiN9w0pYf4ckeCYT9f1r1P9KX5g==" + } } }, "node-pre-gyp": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz", - "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==", + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.15.0.tgz", + "integrity": "sha512-7QcZa8/fpaU/BKenjcaeFF9hLz2+7S9AqyXFhlH/rilsQ/hPZKK32RtR5EQHJElgu+q5RfbJ34KriI79UWaorA==", "requires": { "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.1", + "mkdirp": "^0.5.3", + "needle": "^2.5.0", "nopt": "^4.0.1", "npm-packlist": "^1.1.6", "npmlog": "^4.0.2", "rc": "^1.2.7", "rimraf": "^2.6.1", "semver": "^5.3.0", - "tar": "^4" + "tar": "^4.4.2" } }, "nopt": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", - "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", "requires": { "abbrev": "1", "osenv": "^0.1.4" @@ -1171,17 +1182,26 @@ "optional": true }, "npm-bundled": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.0.6.tgz", - "integrity": "sha512-8/JCaftHwbd//k6y2rEWp6k1wxVfpFzB6t1p825+cUb7Ym2XQfhwIC5KwhrvzZRJu+LtDE585zVaS32+CGtf0g==" + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", + "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "requires": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==" }, "npm-packlist": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.4.tgz", - "integrity": "sha512-zTLo8UcVYtDU3gdeaFu2Xu0n0EvelfHDGuqtNIn5RO7yQj4H1TqNdBc/yZjxnWA0PVB8D3Woyp0i5B43JwQ6Vw==", + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", "requires": { "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" } }, "npmlog": { @@ -1368,13 +1388,6 @@ "ini": "~1.3.0", "minimist": "^1.2.0", "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" - } } }, "readable-stream": { @@ -1505,9 +1518,9 @@ "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" }, "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" }, "simple-websocket": { "version": "9.0.0", @@ -1790,17 +1803,6 @@ "mkdirp": "^0.5.0", "safe-buffer": "^5.1.2", "yallist": "^3.0.3" - }, - "dependencies": { - "minipass": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", - "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", - "requires": { - "safe-buffer": "^5.1.2", - "yallist": "^3.0.0" - } - } } }, "to-array": { @@ -1905,9 +1907,9 @@ "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=" }, "yallist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "yeast": { "version": "0.1.2", diff --git a/package.json b/package.json index f9aae24..6c3531f 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "url": "https://gitlab.com/knotteye/satyr.git" }, "dependencies": { - "bcrypt": "^3.0.6", + "bcrypt": "^5.0.0", "body-parser": "^1.19.0", "cookie-parser": "^1.4.4", "dank-twitch-irc": "^3.2.6", @@ -26,7 +26,7 @@ "irc": "^0.5.2", "jose": "^1.15.1", "mysql": "^2.17.1", - "node-media-server": ">=2.1.3 <3.0.0", + "node-media-server": "^2.2.4", "nunjucks": "^3.2.1", "parse-yaml": "^0.1.0", "recursive-readdir": "^2.2.2", @@ -37,6 +37,6 @@ "typescript": "^3.6.3" }, "devDependencies": { - "@types/node": "^12.7.5" + "@types/node": "^12.12.67" } } From 8caad60a43399c7b1ff37bd25d3d98c37fcbf898 Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 15:29:47 -0500 Subject: [PATCH 14/31] Add functions for generating and using invite codes --- src/api.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/api.ts b/src/api.ts index a803d63..abf2e94 100644 --- a/src/api.ts +++ b/src/api.ts @@ -1,4 +1,5 @@ import * as db from "./database"; +import * as base64id from "base64id"; import { config } from "./config"; import {unlink} from "fs"; @@ -97,4 +98,18 @@ async function getConfig(username: string, all?: boolean): Promise{ return t; } -export { register, update, changepwd, changesk, login, updateChat, deleteVODs, getConfig }; \ No newline at end of file +async function genInvite(user: string): Promise{ + var invitecode: string = base64id.generateId(); + await db.query('INSERT INTO invites (code) VALUES ('+invitecode+')'); + return invitecode; +} + +async function useInvite(code: string): Promise{ + if(typeof(code) !== "string" || code === "") return false; + var result = await db.query('SELECT code FROM invites WHERE code='+db.raw.escape(code)); + if(!result[0] || result[0]['code'] !== code) return false; + await db.query('DELETE FROM invites WHERE code='+db.raw.escape(code)); + return true; +} + +export { register, update, changepwd, changesk, login, updateChat, deleteVODs, getConfig, genInvite, useInvite }; \ No newline at end of file From 9605ff8c92a3abd523b1959d352e456d6266aad5 Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 15:48:39 -0500 Subject: [PATCH 15/31] Add a way to generate invites from the command line. Add database migration script. --- package.json | 5 +++-- src/api.ts | 4 ++-- src/cli.ts | 13 ++++++++++++- src/db/2.ts | 8 ++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 src/db/2.ts diff --git a/package.json b/package.json index 6c3531f..8d685ac 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,10 @@ "author": "knotteye", "scripts": { "start": "ts-node src/index.ts", - "user": "ts-node src/cli.ts", + "cli": "ts-node src/cli.ts", "setup": "sh install/setup.sh", - "migrate": "ts-node src/migrate.ts" + "migrate": "ts-node src/migrate.ts", + "invite": "ts-node src/cli.ts --invite" }, "repository": { "type": "git", diff --git a/src/api.ts b/src/api.ts index abf2e94..55d6437 100644 --- a/src/api.ts +++ b/src/api.ts @@ -98,9 +98,9 @@ async function getConfig(username: string, all?: boolean): Promise{ return t; } -async function genInvite(user: string): Promise{ +async function genInvite(): Promise{ var invitecode: string = base64id.generateId(); - await db.query('INSERT INTO invites (code) VALUES ('+invitecode+')'); + await db.query('INSERT INTO invites (code) VALUES (\"'+invitecode+'\")'); return invitecode; } diff --git a/src/cli.ts b/src/cli.ts index 110ecc6..1109154 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,5 @@ -import * as db from "./database" +import * as db from "./database"; +import * as api from "./api"; import * as flags from "flags"; db.init(); @@ -6,6 +7,7 @@ db.init(); flags.defineString('adduser', '', 'User to add'); flags.defineString('rmuser', '', 'User to remove'); flags.defineString('password', '', 'password to hash'); +flags.defineBoolean('invite', false, 'generate invite code'); flags.parse(); @@ -23,4 +25,13 @@ if(flags.get('rmuser') !== ''){ else console.log("Could not remove user."); process.exit(); }); +} + +if(flags.get('invite')){ + var config = require("./config").config; + api.genInvite().then((r: string) => { + console.log('invite code: '+r); + console.log('Direct the user to https://'+config['satyr']['domain']+'/invite/'+r); + process.exit(); + }); } \ No newline at end of file diff --git a/src/db/2.ts b/src/db/2.ts new file mode 100644 index 0000000..cfe7ae0 --- /dev/null +++ b/src/db/2.ts @@ -0,0 +1,8 @@ +import * as db from "../database"; + +async function run () { + await db.query('CREATE TABLE IF NOT EXISTS invites(code VARCHAR(150))'); + await db.query('INSERT INTO db_meta (version) VALUES (2)'); +} + +export { run } \ No newline at end of file From 67de11e66bab5609ec60c809dca8faeefab7319c Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 16:12:07 -0500 Subject: [PATCH 16/31] Add API handling of invite codes, add web page for inviting users. --- src/api.ts | 13 ++++++++----- src/config.ts | 2 +- src/http.ts | 27 +++++++++++++++++++++++++++ templates/invite.njk | 20 ++++++++++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 templates/invite.njk diff --git a/src/api.ts b/src/api.ts index 55d6437..d421bab 100644 --- a/src/api.ts +++ b/src/api.ts @@ -3,8 +3,8 @@ import * as base64id from "base64id"; import { config } from "./config"; import {unlink} from "fs"; -async function register(name: string, password: string, confirm: string): Promise { - if(!config['satyr']['registration']) return {"error":"registration disabled"}; +async function register(name: string, password: string, confirm: string, invite?: boolean): Promise { + if(!config['satyr']['registration'] && !invite) return {"error":"registration disabled"}; if(name.includes(';') || name.includes(' ') || name.includes('\'')) return {"error":"illegal characters"}; if(password !== confirm) return {"error":"mismatched passwords"}; for(let i=0;i{ return invitecode; } -async function useInvite(code: string): Promise{ +async function validInvite(code: string): Promise{ if(typeof(code) !== "string" || code === "") return false; var result = await db.query('SELECT code FROM invites WHERE code='+db.raw.escape(code)); if(!result[0] || result[0]['code'] !== code) return false; - await db.query('DELETE FROM invites WHERE code='+db.raw.escape(code)); return true; } -export { register, update, changepwd, changesk, login, updateChat, deleteVODs, getConfig, genInvite, useInvite }; \ No newline at end of file +async function useInvite(code: string): Promise{ + if(validInvite(code)) await db.query('DELETE FROM invites WHERE code='+db.raw.escape(code)); +} + +export { register, update, changepwd, changesk, login, updateChat, deleteVODs, getConfig, genInvite, useInvite, validInvite }; \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 4bd6ec0..0fdeaa4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -16,7 +16,7 @@ const config: Object = { domain: '', registration: false, email: null, - restrictedNames: [ 'live', 'user', 'users', 'register', 'login' ], + restrictedNames: [ 'live', 'user', 'users', 'register', 'login', 'invite' ], rootredirect: '/users/live', version: process.env.npm_package_version, }, localconfig['satyr']), diff --git a/src/http.ts b/src/http.ts index b5f9c53..4f2544d 100644 --- a/src/http.ts +++ b/src/http.ts @@ -224,6 +224,21 @@ async function initAPI() { }); }); app.post('/api/register', (req, res) => { + if("invite" in req.body){ + if(api.validInvite(req.body.invite)){ + api.register(req.body.username, req.body.password, req.body.confirm, true).then((result) => { + if(result[0]) return genToken(req.body.username).then((t) => { + res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); + res.json(result); + api.useInvite(req.body.invite); + return; + }); + res.json(result); + }); + } + else res.json({error: "invalid invite code"}); + } + else api.register(req.body.username, req.body.password, req.body.confirm).then( (result) => { if(result[0]) return genToken(req.body.username).then((t) => { res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); @@ -486,6 +501,18 @@ async function initSite(openReg) { } else res.render('login.njk',njkconf); }); + app.get('/invite/:code', (req, res) => { + if(tryDecode(req.cookies.Authorization)) { + res.redirect('/profile'); + } + else res.render('invite.njk',Object.assign({icode: req.params.code}, njkconf)); + }); + app.get('/invite', (req, res) => { + if(tryDecode(req.cookies.Authorization)) { + res.redirect('/profile'); + } + else res.render('invite.njk',Object.assign({icode: ""}, njkconf)); + }); app.get('/register', (req, res) => { if(tryDecode(req.cookies.Authorization) || !openReg) { res.redirect(njkconf.rootredirect); diff --git a/templates/invite.njk b/templates/invite.njk new file mode 100644 index 0000000..9b2b30b --- /dev/null +++ b/templates/invite.njk @@ -0,0 +1,20 @@ +{% extends "base.njk" %} +{% block content %} +

You've been invited to {{ sitename }}

Already registered? Log in here.

 + + + Username:

+ Password:

+ Confirm:

+ Invite Code:


+ +
+ + + {% include "tos.html" %}
+ + +{% endblock %} \ No newline at end of file From eba53c37325c4bbac1044d74a49159287a251ff3 Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 16:16:37 -0500 Subject: [PATCH 17/31] Rework invitation UI a bit, document API changes --- docs/REST.md | 4 +++- src/http.ts | 6 ------ templates/invite.njk | 4 ++-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/REST.md b/docs/REST.md index 3160cf7..c6004a0 100644 --- a/docs/REST.md +++ b/docs/REST.md @@ -91,7 +91,9 @@ Register a new user. **Authentication**: no -**Parameters**: Username, password, confirm +**Parameters**: Username, password, confirm, invite(optional) + +Invite is an optional invite code to bypass disabled registration. **Response**: If successful, returns a json object with the users stream key. Otherwise returns `{error: "error reason"}` diff --git a/src/http.ts b/src/http.ts index 4f2544d..0c0aa6b 100644 --- a/src/http.ts +++ b/src/http.ts @@ -507,12 +507,6 @@ async function initSite(openReg) { } else res.render('invite.njk',Object.assign({icode: req.params.code}, njkconf)); }); - app.get('/invite', (req, res) => { - if(tryDecode(req.cookies.Authorization)) { - res.redirect('/profile'); - } - else res.render('invite.njk',Object.assign({icode: ""}, njkconf)); - }); app.get('/register', (req, res) => { if(tryDecode(req.cookies.Authorization) || !openReg) { res.redirect(njkconf.rootredirect); diff --git a/templates/invite.njk b/templates/invite.njk index 9b2b30b..c181635 100644 --- a/templates/invite.njk +++ b/templates/invite.njk @@ -6,8 +6,8 @@
Username:

Password:

- Confirm:

- Invite Code:


+ Confirm:


+

From acce235812e0181616321d5fdf58032e2ed57ede Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 16:17:15 -0500 Subject: [PATCH 18/31] Increment minor version due to backwards compatible API changes --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 8d685ac..e2b210f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.9.3", + "version": "0.9.4", "description": "A livestreaming server.", "license": "AGPL-3.0", "author": "knotteye", From 1a410a597a499f1ed9d589930e489910f8525329 Mon Sep 17 00:00:00 2001 From: knotteye Date: Tue, 13 Oct 2020 16:29:13 -0500 Subject: [PATCH 19/31] Fix a bug checking the validity of invite codes --- src/http.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/http.ts b/src/http.ts index 0c0aa6b..8eb06dc 100644 --- a/src/http.ts +++ b/src/http.ts @@ -225,18 +225,20 @@ async function initAPI() { }); app.post('/api/register', (req, res) => { if("invite" in req.body){ - if(api.validInvite(req.body.invite)){ - api.register(req.body.username, req.body.password, req.body.confirm, true).then((result) => { - if(result[0]) return genToken(req.body.username).then((t) => { - res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); + api.validInvite(req.body.invite).then((v) => { + if(v){ + api.register(req.body.username, req.body.password, req.body.confirm, true).then((result) => { + if(result[0]) return genToken(req.body.username).then((t) => { + res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); + res.json(result); + api.useInvite(req.body.invite); + return; + }); res.json(result); - api.useInvite(req.body.invite); - return; }); - res.json(result); - }); - } - else res.json({error: "invalid invite code"}); + } + else res.json({error: "invalid invite code"}); + }); } else api.register(req.body.username, req.body.password, req.body.confirm).then( (result) => { From 57d0b0f856737f9fdc56825ca817b7379f3e5a0f Mon Sep 17 00:00:00 2001 From: knotteye Date: Wed, 14 Oct 2020 00:03:45 -0500 Subject: [PATCH 20/31] initial work on client-side templating --- package.json | 3 ++- site/index.html | 37 +++++++++++++++++++++++++++++++++++++ src/http.ts | 9 ++++++++- 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 site/index.html diff --git a/package.json b/package.json index e2b210f..dfb5249 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "cli": "ts-node src/cli.ts", "setup": "sh install/setup.sh", "migrate": "ts-node src/migrate.ts", - "invite": "ts-node src/cli.ts --invite" + "invite": "ts-node src/cli.ts --invite", + "make-templates": "cp node_modules/nunjucks/browser/nunjucks-slim.js site &&nunjucks-precompile -i [\"\\.html$\",\"\\.njk$\"] templates > site/templates.js" }, "repository": { "type": "git", diff --git a/site/index.html b/site/index.html new file mode 100644 index 0000000..b7a72bd --- /dev/null +++ b/site/index.html @@ -0,0 +1,37 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/http.ts b/src/http.ts index 8eb06dc..1576883 100644 --- a/src/http.ts +++ b/src/http.ts @@ -56,11 +56,12 @@ async function init(){ } app.disable('x-powered-by'); //site handlers - await initSite(config['satyr']['registration']); + //await initSite(config['satyr']['registration']); //api handlers await initAPI(); //static files if nothing else matches first app.use(express.static(config['http']['directory'])); + await initFE(); //404 Handler app.use(function (req, res, next) { if(tryDecode(req.cookies.Authorization)) { @@ -73,6 +74,12 @@ async function init(){ server.listen(config['http']['port']); } +async function initFE(){ + app.get('*', (req, res) => { + res.sendFile(process.cwd()+'/'+config['http']['directory']+'/index.html'); + }); +} + async function newNick(socket, skip?: boolean, i?: number) { if(socket.handshake.headers['cookie'] && !skip){ let c = await parseCookie(socket.handshake.headers['cookie']); From 988e3473a74f7a57e1f87a30133197c450318560 Mon Sep 17 00:00:00 2001 From: knotteye Date: Wed, 14 Oct 2020 07:44:19 -0500 Subject: [PATCH 21/31] Big commit. Implement handlers for everything that's currently rendered server side in the client-side frontend. Add compiled templates file to .gitignore, will work out a system for making sure templates are compiled later. Fix a couple bugs in the API and templates. TODO for client-side rendering: Make sure templates get compiled before running the server. Add a config option to switch between server-side and client-side rendering Fancy SPA stuff like intercepting links to render changes without a page-reload --- .gitignore | 1 + package-lock.json | 7 +- package.json | 2 +- site/index.html | 40 ++++------- site/index.js | 141 +++++++++++++++++++++++++++++++++++++++ src/api.ts | 4 +- src/http.ts | 11 +++ templates/managevods.njk | 2 +- 8 files changed, 177 insertions(+), 31 deletions(-) create mode 100644 site/index.js diff --git a/.gitignore b/.gitignore index 8c8153b..4714669 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ config/**/* !config/.gitkeep install/db_setup.sql build/** +site/templates.js \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 78361b5..e6e1e5a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.9.3", + "version": "0.9.4", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -963,6 +963,11 @@ "asn1.js": "^5.2.0" } }, + "jwt-decode": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.0.0.tgz", + "integrity": "sha512-RBQv2MTm3FNKQkdzhEyQwh5MbdNgMa+FyIJIK5RMWEn6hRgRHr7j55cRxGhRe6vGJDElyi6f6u/yfkP7AoXddA==" + }, "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", diff --git a/package.json b/package.json index dfb5249..f7f4fab 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "setup": "sh install/setup.sh", "migrate": "ts-node src/migrate.ts", "invite": "ts-node src/cli.ts --invite", - "make-templates": "cp node_modules/nunjucks/browser/nunjucks-slim.js site &&nunjucks-precompile -i [\"\\.html$\",\"\\.njk$\"] templates > site/templates.js" + "make-templates": "nunjucks-precompile -i [\"\\.html$\",\"\\.njk$\"] templates > site/templates.js" }, "repository": { "type": "git", diff --git a/site/index.html b/site/index.html index b7a72bd..af4c9bb 100644 --- a/site/index.html +++ b/site/index.html @@ -1,37 +1,23 @@ + + + + - + \ No newline at end of file diff --git a/site/index.js b/site/index.js new file mode 100644 index 0000000..c94fc3a --- /dev/null +++ b/site/index.js @@ -0,0 +1,141 @@ +async function render(){ + var context = await getContext(); + switch(window.location.pathname){ + //nothing but context + case (window.location.pathname.match(/^\/about\/?$/) || {}).input: + document.body.innerHTML = nunjucks.render('about.njk', context); + break; + case (window.location.pathname.match(/^\/login\/?$/) || {}).input: + document.body.innerHTML = nunjucks.render('login.njk', context); + break; + case (window.location.pathname.match(/^\/register\/?$/) || {}).input: + if(!context.registration) window.location = '/'; + document.body.innerHTML = nunjucks.render('registration.njk', context); + break; + case (window.location.pathname.match(/^\/changepwd\/?$/) || {}).input: + document.body.innerHTML = nunjucks.render('changepwd.njk', context); + break; + case (window.location.pathname.match(/^\/chat\/?$/) || {}).input: + document.body.innerHTML = nunjucks.render('chat.html', context); + break; + case (window.location.pathname.match(/^\/help\/?$/) || {}).input: + document.body.innerHTML = nunjucks.render('help.njk', context); + break; + //need to hit the API + case (window.location.pathname.match(/^\/users\/live\/?$/) || {}).input: + var list = JSON.parse(await makeRequest("POST", "/api/users/live", JSON.stringify({num: 50}))); + document.body.innerHTML = nunjucks.render('live.njk', Object.assign({list: list.users}, context)); + break; + case (window.location.pathname.match(/^\/users\/?$/) || {}).input: + var list = JSON.parse(await makeRequest("POST", "/api/users/all", JSON.stringify({num: 50}))); + document.body.innerHTML = nunjucks.render('list.njk', Object.assign({list: list.users}, context)); + break; + case (window.location.pathname.match(/^\/profile\/chat\/?$/) || {}).input: + if(!context.auth.name) window.location = '/login'; + var config = JSON.parse(await makeRequest("GET", '/api/'+context.auth.name+'/config')); + config = { + integ: { + twitch: config.twitch, + xmpp: config.xmpp, + irc: config.irc, + discord: config.discord + } + }; + document.body.innerHTML = nunjucks.render('chat_integ.njk', Object.assign(config, context)); + break; + case (window.location.pathname.match(/^\/profile\/?$/) || {}).input: + if(!context.auth.name) window.location = '/login'; + var config = JSON.parse(await makeRequest("GET", '/api/'+context.auth.name+'/config')); + config = { + meta: { + title: config.title, + about: config.about + }, + rflag: {record_flag: config.record_flag}, + twitch: config.twitch_mirror + }; + document.body.innerHTML = nunjucks.render('profile.njk', Object.assign(config, context)); + break; + //parsing slugs + case (window.location.pathname.match(/^\/invite\//) || {}).input: // /invite/:code + document.body.innerHTML = nunjucks.render('invite.njk', Object.assign({icode: window.location.pathname.substring(8)}, context)); + break; + //slugs and API + case (window.location.pathname.match(/^\/users\/.+\/?$/) || {}).input: // /users/:user + if(window.location.pathname.substring(window.location.pathname.length - 1).indexOf('/') !== -1) + var usr = window.location.pathname.substring(7, window.location.pathname.length - 1); + else var usr = window.location.pathname.substring(7); + var config = JSON.parse(await makeRequest("GET", '/api/'+usr+'/config')); + if(!config.title){document.body.innerHTML = nunjucks.render('404.njk', context); break;} + document.body.innerHTML = nunjucks.render('user.njk', Object.assign({about: config.about, title: config.title, username: config.username}, context)); + break; + case (window.location.pathname.match(/^\/vods\/.+\/manage\/?$/) || {}).input: // /vods/:user/manage + var usr = window.location.pathname.substring(6, (window.location.pathname.length - 7)); + if(context.auth.name !== usr) window.location = '/vods/'+usr; + var vods = JSON.parse(await makeRequest("GET", '/api/'+usr+'/vods')); + document.body.innerHTML = nunjucks.render('managevods.njk', Object.assign({user: usr, list: vods.vods.filter(fn => fn.name.endsWith('.mp4'))}, context)); + break; + case (window.location.pathname.match(/^\/vods\/.+\/?$/) || {}).input: // /vods/:user + if(window.location.pathname.substring(window.location.pathname.length - 1).indexOf('/') !== -1) + var usr = window.location.pathname.substring(6, window.location.pathname.length - 1); + else var usr = window.location.pathname.substring(6); + var vods = JSON.parse(await makeRequest("GET", '/api/'+usr+'/vods')); + document.body.innerHTML = nunjucks.render('vods.njk', Object.assign({user: usr, list: vods.vods.filter(fn => fn.name.endsWith('.mp4'))}, context)); + break; + //root + case "/": + window.location = '/users/live'; + break; + case "": + window.location = '/users/live'; + break; + //404 + default: + document.body.innerHTML = nunjucks.render('404.njk', context); + } +} + +async function getContext(){ + var info = JSON.parse(await makeRequest('GET', '/api/instance/info')); + info.sitename = info.name; + info.name = null; + info.auth = { + is: document.cookie.match(/^(.*;)?\s*X-Auth-As\s*=\s*[^;]+(.*)?$/) !== null, + name: parseCookie(document.cookie)['X-Auth-As'] + } + return info; +} + +function makeRequest(method, url, payload) { + return new Promise(function (resolve, reject) { + let xhr = new XMLHttpRequest(); + xhr.open(method, url); + xhr.onload = function () { + if (this.status >= 200 && this.status < 300) { + resolve(xhr.response); + } else { + reject({ + status: this.status, + statusText: xhr.statusText + }); + } + }; + xhr.onerror = function () { + reject({ + status: this.status, + statusText: xhr.statusText + }); + }; + !payload ? xhr.send() : xhr.send(payload); + }); +} + +function parseCookie(c){ + if(typeof(c) !== 'string' || !c.includes('=')) return {}; + return Object.assign({[c.split('=')[0].trim()]:c.split('=')[1].split(';')[0].trim()}, parseCookie(c.split(/;(.+)/)[1])); +} + +function handleLoad() { + var r = JSON.parse(document.getElementById('responseFrame').contentDocument.documentElement.textContent).success + if (typeof(r) !== 'undefined') window.location.href = '/profile' +} \ No newline at end of file diff --git a/src/api.ts b/src/api.ts index d421bab..27b0d68 100644 --- a/src/api.ts +++ b/src/api.ts @@ -87,9 +87,11 @@ async function getConfig(username: string, all?: boolean): Promise{ let users = await db.query('SELECT stream_key,record_flag FROM users WHERE username='+db.raw.escape(username)); if(users[0]) Object.assign(t, users[0]); let usermeta = await db.query('SELECT title,about FROM user_meta WHERE username='+db.raw.escape(username)); - if(usermeta[0]) Object.assign(t, users[0]); + if(usermeta[0]) Object.assign(t, usermeta[0]); let ci = await db.query('SELECT irc,xmpp,twitch,discord FROM chat_integration WHERE username='+db.raw.escape(username)); if(ci[0]) Object.assign(t, ci[0]); + let tw = await db.query('SELECT enabled,twitch_key FROM twitch_mirror WHERE username='+db.raw.escape(username)); + if(tw[0]) t['twitch_mirror'] = Object.assign({}, tw[0]); } else { let um = await db.query('SELECT title,about FROM user_meta WHERE username='+db.raw.escape(username)); diff --git a/src/http.ts b/src/http.ts index 1576883..967a1f3 100644 --- a/src/http.ts +++ b/src/http.ts @@ -75,6 +75,15 @@ async function init(){ } async function initFE(){ + app.get('/', (req, res) => { + res.redirect(config['satyr']['rootredirect']); + }); + app.get('/nunjucks-slim.js', (req, res) => { + res.sendFile(process.cwd()+'/node_modules/nunjucks/browser/nunjucks-slim.js'); + }); + app.get('/chat', (req, res) => { + res.sendFile(process.cwd()+'/templates/chat.html'); + }); app.get('*', (req, res) => { res.sendFile(process.cwd()+'/'+config['http']['directory']+'/index.html'); }); @@ -368,6 +377,7 @@ async function initAPI() { if(req.cookies.Authorization) validToken(req.cookies.Authorization).then((t) => { if(t) { if(t['exp'] - 86400 < Math.floor(Date.now() / 1000)){ + res.cookie('X-Auth-As', t['username'], {maxAge: 604800000, httpOnly: false, sameSite: 'Lax'}); return genToken(t['username']).then((t) => { res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); res.json({success:""}); @@ -389,6 +399,7 @@ async function initAPI() { if(!result){ genToken(req.body.username).then((t) => { res.cookie('Authorization', t, {maxAge: 604800000, httpOnly: true, sameSite: 'Lax'}); + res.cookie('X-Auth-As', req.body.username, {maxAge: 604800000, httpOnly: false, sameSite: 'Lax'}); res.json({success:""}); }) } diff --git a/templates/managevods.njk b/templates/managevods.njk index 20928b1..0e8f2dc 100644 --- a/templates/managevods.njk +++ b/templates/managevods.njk @@ -8,7 +8,7 @@ {% else %} No recordings found! {% endeach %} - +
{% endblock %} \ No newline at end of file From 4ec89d71f8a1616798dbc51106e2f306351fac86 Mon Sep 17 00:00:00 2001 From: knotteye Date: Wed, 14 Oct 2020 07:51:26 -0500 Subject: [PATCH 22/31] Bump major version. There were some breaking changes in there somewhere --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f7f4fab..e921a3e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.9.4", + "version": "0.10.0", "description": "A livestreaming server.", "license": "AGPL-3.0", "author": "knotteye", From 961b5fe648429e11c937c14d15c45fc5efcb7b23 Mon Sep 17 00:00:00 2001 From: knotteye Date: Fri, 16 Oct 2020 21:31:23 -0500 Subject: [PATCH 23/31] Add config option to turn server side rendering off. Ensure templates are precompiled before starting the server. --- README.md | 6 ++++-- install/config.example.yml | 1 + src/cleanup.ts | 14 ++++++++++++-- src/config.ts | 5 ++++- src/index.ts | 2 +- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2018270..db204c1 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,13 @@ Follow the instructions after setup runs. ```bash npm run start ``` -You can also run this to skip checking the database version on startup. +You can also skip checking the database version and compiling templates (if you don't use server-side rendering) on startup. ```bash -npm run start -- --skip-migrate +npm run start -- --skip-migrate --skip-compile # don't forget to migrate manually when you update npm run migrate +# and compile templates after any changes +npm run make-templates ``` ## Contributing diff --git a/install/config.example.yml b/install/config.example.yml index c67d16b..be1a5a0 100644 --- a/install/config.example.yml +++ b/install/config.example.yml @@ -14,6 +14,7 @@ rtmp: http: # uncomment to set HSTS when SSL is ready #hsts: true + server_side_render: false database: user: '' diff --git a/src/cleanup.ts b/src/cleanup.ts index 56c9dc9..4a3fd58 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -1,8 +1,9 @@ import * as db from "./database"; import {readdirSync} from "fs"; +import { execSync } from "child_process"; -async function init(m?: boolean) { - if(!m){ +async function init() { + if(process.argv.indexOf('--skip-migrate') === -1){ console.log('Checking database version.'); var tmp: string[] = await db.query('show tables like \"db_meta\"'); if(tmp.length === 0){ @@ -17,6 +18,15 @@ async function init(m?: boolean) { else { console.log('Skipping database version check.'); } + + if(!require('./config').config['http']['server_side_render'] && process.argv.indexOf('--skip-compile') === -1) { + console.log("Compiling templates for client-side frontend."); + execSync(process.cwd()+'/node_modules/.bin/nunjucks-precompile -i [\"\\.html$\",\"\\.njk$\"] templates > site/templates.js'); + } + else if(!require('./config').config['http']['server_side_render']){ + console.log("Skipped compiling templates for client-side frontend."); + } + //If satyr is restarted in the middle of a stream //it causes problems //Live flags in the database stay live diff --git a/src/config.ts b/src/config.ts index 0fdeaa4..d9edb2e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,7 +36,10 @@ const config: Object = { ping: 30, ping_timeout: 60 }, localconfig['rtmp']), http: Object.assign({ - hsts: false, directory: './site', port: 8000 + hsts: false, + directory: './site', + port: 8000, + server_side_render: true }, localconfig['http']), media: Object.assign({ record: false, diff --git a/src/index.ts b/src/index.ts index 889c20f..2573948 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,7 +7,7 @@ import { config } from "./config"; async function run() { await initDB(); - await clean(process.argv.indexOf('--skip-migrate') !== -1); + await clean(); await initHTTP(); await initRTMP(); await initChat(); From 95837beaf7818f3159d40fb630bc43a436728d8e Mon Sep 17 00:00:00 2001 From: knotteye Date: Fri, 16 Oct 2020 21:57:24 -0500 Subject: [PATCH 24/31] Make server side rendering fully configurable --- src/http.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/http.ts b/src/http.ts index 967a1f3..f83b61b 100644 --- a/src/http.ts +++ b/src/http.ts @@ -12,9 +12,6 @@ import * as chatInteg from "./chat"; import { config } from "./config"; import { readdir, readFileSync, writeFileSync } from "fs"; import { JWT, JWK } from "jose"; -import { strict } from "assert"; -import { parse } from "path"; -import { isBuffer } from "util"; const app = express(); const server = http.createServer(app); @@ -55,12 +52,15 @@ async function init(){ }); } app.disable('x-powered-by'); - //site handlers - //await initSite(config['satyr']['registration']); - //api handlers + //server-side site routes + if(config['http']['server_side_render']) + await initSite(config['satyr']['registration']); + //api routes await initAPI(); - //static files if nothing else matches first + //static files if nothing else matches app.use(express.static(config['http']['directory'])); + //client-side site routes + if(!config['http']['server_side_render']) await initFE(); //404 Handler app.use(function (req, res, next) { From 54a891dac1eea53cdf36187b8a084518e76dcafb Mon Sep 17 00:00:00 2001 From: knotteye Date: Fri, 16 Oct 2020 22:25:24 -0500 Subject: [PATCH 25/31] Update documentation --- docs/REST.md | 2 +- templates/base.njk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/REST.md b/docs/REST.md index c6004a0..9058efb 100644 --- a/docs/REST.md +++ b/docs/REST.md @@ -115,7 +115,7 @@ Obtain a signed json web token for authentication **Response**: If succesful, will return `{success: ""}` or `{success: "already verified"}` if the JWT provided is too early to be renewed. If unsuccesful, will return `{error: "invalid password"}` or `{error: "Username or Password Incorrect"}` depending on the authentication method. Note that if a JWT is available, the parameters will be ignored. -**Notes**: I've already listed nearly every response. My final note is that the JWT is set as the cookie 'Authorization', not returned in the response. +**Notes**: The returned JWT is set as the cookie httponly 'Authorization'. It will also return a non httponly cookie X-Auth-As with the username of the authenticated user. ## /api/user/update diff --git a/templates/base.njk b/templates/base.njk index 9d57b04..f832bc3 100644 --- a/templates/base.njk +++ b/templates/base.njk @@ -6,7 +6,7 @@ {{ sitename }} - + \ No newline at end of file diff --git a/site/index.js b/site/index.js index c94fc3a..8a06b30 100644 --- a/site/index.js +++ b/site/index.js @@ -1,36 +1,44 @@ -async function render(){ +async function render(path){ var context = await getContext(); - switch(window.location.pathname){ + switch(path){ //nothing but context - case (window.location.pathname.match(/^\/about\/?$/) || {}).input: + case (path.match(/^\/about\/?$/) || {}).input: document.body.innerHTML = nunjucks.render('about.njk', context); + modifyLinks(); break; - case (window.location.pathname.match(/^\/login\/?$/) || {}).input: + case (path.match(/^\/login\/?$/) || {}).input: document.body.innerHTML = nunjucks.render('login.njk', context); + modifyLinks(); break; - case (window.location.pathname.match(/^\/register\/?$/) || {}).input: + case (path.match(/^\/register\/?$/) || {}).input: if(!context.registration) window.location = '/'; document.body.innerHTML = nunjucks.render('registration.njk', context); + modifyLinks(); break; - case (window.location.pathname.match(/^\/changepwd\/?$/) || {}).input: + case (path.match(/^\/changepwd\/?$/) || {}).input: document.body.innerHTML = nunjucks.render('changepwd.njk', context); + modifyLinks(); break; - case (window.location.pathname.match(/^\/chat\/?$/) || {}).input: + case (path.match(/^\/chat\/?$/) || {}).input: document.body.innerHTML = nunjucks.render('chat.html', context); + modifyLinks(); break; - case (window.location.pathname.match(/^\/help\/?$/) || {}).input: + case (path.match(/^\/help\/?$/) || {}).input: document.body.innerHTML = nunjucks.render('help.njk', context); + modifyLinks(); break; //need to hit the API - case (window.location.pathname.match(/^\/users\/live\/?$/) || {}).input: + case (path.match(/^\/users\/live\/?$/) || {}).input: var list = JSON.parse(await makeRequest("POST", "/api/users/live", JSON.stringify({num: 50}))); document.body.innerHTML = nunjucks.render('live.njk', Object.assign({list: list.users}, context)); + modifyLinks(); break; - case (window.location.pathname.match(/^\/users\/?$/) || {}).input: + case (path.match(/^\/users\/?$/) || {}).input: var list = JSON.parse(await makeRequest("POST", "/api/users/all", JSON.stringify({num: 50}))); document.body.innerHTML = nunjucks.render('list.njk', Object.assign({list: list.users}, context)); + modifyLinks(); break; - case (window.location.pathname.match(/^\/profile\/chat\/?$/) || {}).input: + case (path.match(/^\/profile\/chat\/?$/) || {}).input: if(!context.auth.name) window.location = '/login'; var config = JSON.parse(await makeRequest("GET", '/api/'+context.auth.name+'/config')); config = { @@ -42,8 +50,9 @@ async function render(){ } }; document.body.innerHTML = nunjucks.render('chat_integ.njk', Object.assign(config, context)); + modifyLinks(); break; - case (window.location.pathname.match(/^\/profile\/?$/) || {}).input: + case (path.match(/^\/profile\/?$/) || {}).input: if(!context.auth.name) window.location = '/login'; var config = JSON.parse(await makeRequest("GET", '/api/'+context.auth.name+'/config')); config = { @@ -55,43 +64,49 @@ async function render(){ twitch: config.twitch_mirror }; document.body.innerHTML = nunjucks.render('profile.njk', Object.assign(config, context)); + modifyLinks(); break; //parsing slugs - case (window.location.pathname.match(/^\/invite\//) || {}).input: // /invite/:code - document.body.innerHTML = nunjucks.render('invite.njk', Object.assign({icode: window.location.pathname.substring(8)}, context)); + case (path.match(/^\/invite\//) || {}).input: // /invite/:code + document.body.innerHTML = nunjucks.render('invite.njk', Object.assign({icode: path.substring(8)}, context)); + modifyLinks(); break; //slugs and API - case (window.location.pathname.match(/^\/users\/.+\/?$/) || {}).input: // /users/:user - if(window.location.pathname.substring(window.location.pathname.length - 1).indexOf('/') !== -1) - var usr = window.location.pathname.substring(7, window.location.pathname.length - 1); - else var usr = window.location.pathname.substring(7); + case (path.match(/^\/users\/.+\/?$/) || {}).input: // /users/:user + if(path.substring(path.length - 1).indexOf('/') !== -1) + var usr = path.substring(7, path.length - 1); + else var usr = path.substring(7); var config = JSON.parse(await makeRequest("GET", '/api/'+usr+'/config')); if(!config.title){document.body.innerHTML = nunjucks.render('404.njk', context); break;} document.body.innerHTML = nunjucks.render('user.njk', Object.assign({about: config.about, title: config.title, username: config.username}, context)); + modifyLinks(); break; - case (window.location.pathname.match(/^\/vods\/.+\/manage\/?$/) || {}).input: // /vods/:user/manage - var usr = window.location.pathname.substring(6, (window.location.pathname.length - 7)); + case (path.match(/^\/vods\/.+\/manage\/?$/) || {}).input: // /vods/:user/manage + var usr = path.substring(6, (path.length - 7)); if(context.auth.name !== usr) window.location = '/vods/'+usr; var vods = JSON.parse(await makeRequest("GET", '/api/'+usr+'/vods')); document.body.innerHTML = nunjucks.render('managevods.njk', Object.assign({user: usr, list: vods.vods.filter(fn => fn.name.endsWith('.mp4'))}, context)); + modifyLinks(); break; - case (window.location.pathname.match(/^\/vods\/.+\/?$/) || {}).input: // /vods/:user - if(window.location.pathname.substring(window.location.pathname.length - 1).indexOf('/') !== -1) - var usr = window.location.pathname.substring(6, window.location.pathname.length - 1); - else var usr = window.location.pathname.substring(6); + case (path.match(/^\/vods\/.+\/?$/) || {}).input: // /vods/:user + if(path.substring(path.length - 1).indexOf('/') !== -1) + var usr = path.substring(6, path.length - 1); + else var usr = path.substring(6); var vods = JSON.parse(await makeRequest("GET", '/api/'+usr+'/vods')); document.body.innerHTML = nunjucks.render('vods.njk', Object.assign({user: usr, list: vods.vods.filter(fn => fn.name.endsWith('.mp4'))}, context)); + modifyLinks(); break; //root case "/": - window.location = '/users/live'; + render('/users/live'); break; case "": - window.location = '/users/live'; + render('/users/live'); break; //404 default: document.body.innerHTML = nunjucks.render('404.njk', context); + modifyLinks(); } } @@ -138,4 +153,18 @@ function parseCookie(c){ function handleLoad() { var r = JSON.parse(document.getElementById('responseFrame').contentDocument.documentElement.textContent).success if (typeof(r) !== 'undefined') window.location.href = '/profile' +} + +function modifyLinks() { + for (var ls = document.links, numLinks = ls.length, i=0; i Date: Fri, 21 Aug 2020 06:50:40 -0500 Subject: [PATCH 27/31] Add the beginnings of the ability to cluster RTMP servers. It looks like there won't be a way to reliably play RTMP streams like this without digging into node-media-server code. For now that means clustering will have the drawback of being able to do DASH only. Still need to add a config option and reliable recording. --- src/cluster.ts | 272 ++++++++++++++++++++++++++++++++++++++++++++++++ src/database.ts | 9 +- 2 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 src/cluster.ts diff --git a/src/cluster.ts b/src/cluster.ts new file mode 100644 index 0000000..5de1356 --- /dev/null +++ b/src/cluster.ts @@ -0,0 +1,272 @@ +import * as cluster from 'cluster'; +import * as net from 'net'; +import * as NodeRtmpSession from '../node_modules/node-media-server/node_rtmp_session'; +import * as logger from '../node_modules/node-media-server/node_core_logger'; +import * as dirty from "dirty"; +import { mkdir, fstat, access } from "fs"; +import * as strf from "strftime"; +import * as ctx from '../node_modules/node-media-server/node_core_ctx'; +import * as db from "./database"; +import {config} from "./config"; +import { messageRateLimitPresets } from 'dank-twitch-irc'; + +const {readValue, writeValue} = require('@mediafish/amf0'); + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +const { exec, execFile } = require('child_process'); + +const keystore = dirty(); +const num_processes = require('os').cpus().length; +const workerMap = {}; + +if (cluster.isMaster) { + //master logic + + //store workers in here + var workers = []; + + // Helper function for spawning worker at index 'i'. + var spawn = function(i) { + workers[i] = cluster.fork(); + workers[i].on('message', (msg) => { + handleMsgMaster(msg, i) + }); + + // Restart worker on exit + workers[i].on('exit', function(code, signal) { + console.log('[RTMP Cluster MASTER] Respawning Worker', i); + spawn(i); + }); + }; + + // Spawn initial workers + for (var i = 0; i < num_processes; i++) { + spawn(i); + } + + var nextWorker: number = 0; + + //TODO assign incoming connections correctly + + var server = net.createServer({ pauseOnConnect: true }, function(connection) { + if(nextWorker >= workers.length) nextWorker = 0; + var worker = workers[nextWorker]; + /*connection.on('data', (chunk) => { + const buff = Buffer.from(chunk); + let offset, value, array = []; + while(true) { + [offset, value] = readValue(buff, offset); + console.log(JSON.stringify(value, null, 4)); + array.push(value); + } + });*/ + worker.send('rtmp-session:connection', connection); //send connection to worker + }).listen(config['rtmp']['port']); + + console.log('[RTMP Cluster MASTER] Master Ready.'); +} else { + + //worker logic + + //we need our own database pool since we can't share memory anyone else + db.initRTMPCluster(); + + const rtmpcfg = { + logType: 0, + rtmp: Object.assign({port: 1936}, config['rtmp']) + }; + + // creating the rtmp server + var serv = net.createServer((socket) => { + let session = new NodeRtmpSession(rtmpcfg, socket); + session.run(); + }).listen(1936); + logger.setLogType(0); + + // RTMP Server Logic + newRTMPListener('postPublish', (id, StreamPath, args) =>{ + console.log(`[RTMP Cluster WORKER ${process.pid}] Publish Hook for stream: ${id}`); + let session = getRTMPSession(id); + let app: string = StreamPath.split("/")[1]; + let key: string = StreamPath.split("/")[2]; + //disallow urls not formatted exactly right + if (StreamPath.split("/").length !== 3 || key.includes(' ')){ + console.log(`[RTMP Cluster WORKER ${process.pid}] Malformed URL, closing connection for stream: ${id}`); + session.reject(); + return false; + } + if(app !== config['media']['privateEndpoint']){ + //app isn't at public endpoint if we've reached this point + console.log(`[RTMP Cluster WORKER ${process.pid}] Wrong endpoint, rejecting stream: ${id}`); + session.reject(); + return false; + } + //if the url is formatted correctly and the user is streaming to the correct private endpoint + //grab the username from the database and redirect the stream there if the key is valid + //otherwise kill the session + db.query('select username,record_flag from users where stream_key='+db.raw.escape(key)+' limit 1').then(async (results) => { + if(results[0]){ + //transcode to mpd after making sure directory exists + keystore[results[0].username] = key; + mkdir(config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username, { recursive : true }, ()=>{;}); + while(true){ + if(session.audioCodec !== 0 && session.videoCodec !== 0){ + transCommand(results[0].username, key).then((r) => { + execFile(config['media']['ffmpeg'], r, {maxBuffer: Infinity}, (err, stdout, stderr) => { + /*console.log(err); + console.log(stdout); + console.log(stderr);*/ + }); + }); + break; + } + await sleep(300); + } + if(results[0].record_flag && config['media']['record']){ + console.log(`[RTMP Cluster WORKER ${process.pid}] Initiating recording for stream: ${id}`); + mkdir(config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username, { recursive : true }, (err) => { + if (err) throw err; + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username+'/'+strf('%d%b%Y-%H%M')+'.mp4'], { + detached : true, + stdio : 'inherit', + maxBuffer: Infinity + }).unref(); + //spawn an ffmpeg process to record the stream, then detach it completely + //ffmpeg can then (probably) finalize the recording if satyr crashes mid-stream + }); + } + else { + console.log(`[RTMP Cluster WORKER ${process.pid}] Skipping recording for stream: ${id}`); + } + db.query('update user_meta set live=true where username=\''+results[0].username+'\' limit 1'); + console.log(`[RTMP Cluster WORKER ${process.pid}] Stream key ok for stream: ${id}`); + //notify master process that we're handling the stream for this user + process.send({type: 'handle-publish', name:results[0].username}); + } + else{ + console.log(`[RTMP Cluster WORKER ${process.pid}] Invalid stream key for stream: ${id}`); + session.reject(); + } + }); + }); + + newRTMPListener('donePublish', (id, StreamPath, args) => { + let app: string = StreamPath.split("/")[1]; + let key: string = StreamPath.split("/")[2]; + if(app === config['media']['privateEndpoint']) { + db.query('update user_meta,users set user_meta.live=false where users.stream_key='+db.raw.escape(key)); + db.query('select username from users where stream_key='+db.raw.escape(key)+' limit 1').then(async (results) => { + if(results[0]) keystore.rm(results[0].username); + //notify master process that we're no longer handling the stream for this user + process.send({type: 'handle-publish-done', name:results[0].username}); + }); + } + }); + + newRTMPListener('prePlay', (id, StreamPath, args) => { + let session = getRTMPSession(id); + let app: string = StreamPath.split("/")[1]; + let key: string = StreamPath.split("/")[2]; + //correctly formatted urls again + if (StreamPath.split("/").length !== 3){ + console.log("[NodeMediaServer] Malformed URL, closing connection for stream:",id); + session.reject(); + return false; + } + //localhost can play from whatever endpoint + //other clients must use private endpoint + if(app !== config['media']['publicEndpoint'] && !session.isLocal) { + console.log("[NodeMediaServer] Non-local Play from private endpoint, rejecting client:",id); + session.reject(); + return false; + } + //rewrite playpath to private endpoint serverside + //(hopefully) + if(app === config['media']['publicEndpoint']) { + if(keystore[key]){ + session.playStreamPath = '/'+config['media']['privateEndpoint']+'/'+keystore[key]; + return true; + } + //here the client is asking for a valid stream that we don't have + //so we are going to ask the master process for it + else session.reject(); + } + }); + + //recieve messages from master + process.on('message', function(message, connection) { + if (message === 'rtmp-session:connection') { + // Emulate a connection event on the server by emitting the + // event with the connection the master sent us. + serv.emit('connection', connection); + connection.resume(); + return; + } + if(message['type'] === 'stream-request:h') { + if(!message['available']) + getRTMPSession(message['id']).reject(); + } + }); + console.log(`[RTMP Cluster WORKER ${process.pid}] Worker Ready.`); +} + +function newRTMPListener(eventName, listener) { + ctx.nodeEvent.on(eventName, listener); +} + +function getRTMPSession(id) { + return ctx.sessions.get(id); +} + +async function transCommand(user: string, key: string): Promise{ + let args: string[] = ['-loglevel', 'fatal', '-y']; + if(config['transcode']['inputflags'] !== null && config['transcode']['inputflags'] !== "") args = args.concat(config['transcode']['inputflags'].split(" ")); + args = args.concat(['-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-movflags', '+faststart']); + if(config['transcode']['adaptive']===true && config['transcode']['variants'] > 1) { + for(let i=0;i 51 ? 51 : Math.floor(18 + (i * 7)); + args = args.concat(['-crf:'+i, ''+crf]); + } + for(let i=1;i= workers.length) nextWorker = 0; + } + if(msg['type'] === 'handle-publish-done'){ + workerMap[msg['name']] = undefined; + } + if(msg['type'] === 'stream-request:h'){ + if(workerMap[msg['key']] !== undefined){ + workers[index].send({type: 'stream-request:h', id: msg['id'], key: msg['key'], available: true}); + } + else { + workers[index].send({type: 'stream-request:h', id: msg['id'], key: msg['key'], available: false}); + } + } +} \ No newline at end of file diff --git a/src/database.ts b/src/database.ts index 1ee8a17..0d733e7 100644 --- a/src/database.ts +++ b/src/database.ts @@ -12,6 +12,13 @@ function init (){ console.log('Connected to database.'); } +function initRTMPCluster(){ + let cfg = config['database']; + cfg['connectionLimit'] = Math.floor(config['database']['connectionLimit'] / require('os').cpus().length); + raw = mysql.createPool(cfg); + cryptoconfig = config['crypto']; +} + async function addUser(name: string, password: string){ //does not respect registration setting in config if(password === '') return false; @@ -63,4 +70,4 @@ async function hash(pwd){ return await bcrypt.hash(pwd, cryptoconfig['saltRounds']); } -export { query, raw, init, addUser, rmUser, validatePassword, hash, genKey }; \ No newline at end of file +export { query, raw, init, addUser, rmUser, validatePassword, hash, genKey, initRTMPCluster }; \ No newline at end of file From 7806b34cfa4f0e0b8aa3efb8e7058ae7bcabf38c Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 17 Oct 2020 01:03:20 -0500 Subject: [PATCH 28/31] Remove unused code --- src/cluster.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/cluster.ts b/src/cluster.ts index 5de1356..b9288fa 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -8,9 +8,6 @@ import * as strf from "strftime"; import * as ctx from '../node_modules/node-media-server/node_core_ctx'; import * as db from "./database"; import {config} from "./config"; -import { messageRateLimitPresets } from 'dank-twitch-irc'; - -const {readValue, writeValue} = require('@mediafish/amf0'); const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); const { exec, execFile } = require('child_process'); @@ -51,15 +48,6 @@ if (cluster.isMaster) { var server = net.createServer({ pauseOnConnect: true }, function(connection) { if(nextWorker >= workers.length) nextWorker = 0; var worker = workers[nextWorker]; - /*connection.on('data', (chunk) => { - const buff = Buffer.from(chunk); - let offset, value, array = []; - while(true) { - [offset, value] = readValue(buff, offset); - console.log(JSON.stringify(value, null, 4)); - array.push(value); - } - });*/ worker.send('rtmp-session:connection', connection); //send connection to worker }).listen(config['rtmp']['port']); From 1ae7128b9dbe2bf653e347c8601e8807c61eb6f8 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 17 Oct 2020 01:07:29 -0500 Subject: [PATCH 29/31] Update server code --- src/cluster.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/cluster.ts b/src/cluster.ts index b9288fa..72f4fa2 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -127,6 +127,16 @@ if (cluster.isMaster) { console.log(`[RTMP Cluster WORKER ${process.pid}] Skipping recording for stream: ${id}`); } db.query('update user_meta set live=true where username=\''+results[0].username+'\' limit 1'); + db.query('SELECT twitch_key,enabled from twitch_mirror where username='+db.raw.escape(results[0].username)+' limit 1').then(async (tm) => { + if(!tm[0]['enabled'] || !config['twitch_mirror']['enabled'] || !config['twitch_mirror']['ingest']) return; + console.log('[NodeMediaServer] Mirroring to twitch for stream:',id) + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { + detached: true, + stdio : 'inherit', + maxBuffer: Infinity + }).unref(); + }); + console.log('[NodeMediaServer] Stream key ok for stream:',id); console.log(`[RTMP Cluster WORKER ${process.pid}] Stream key ok for stream: ${id}`); //notify master process that we're handling the stream for this user process.send({type: 'handle-publish', name:results[0].username}); From 80cf01ef30ccef24264fe932ccea411041a89302 Mon Sep 17 00:00:00 2001 From: knotteye Date: Sat, 17 Oct 2020 01:53:33 -0500 Subject: [PATCH 30/31] Add unique ports per worker so that ffmpeg can reliably record. --- package-lock.json | 12 ++++++------ package.json | 1 + src/cluster.ts | 31 +++++++++++++++++++++++++------ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index e6e1e5a..c5e1872 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "satyr", - "version": "0.9.4", + "version": "0.10.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -950,6 +950,11 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "optional": true }, + "is-port-available": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/is-port-available/-/is-port-available-0.1.5.tgz", + "integrity": "sha512-/r7UZAQtfgDFdhxzM71jG0mkC4oSRA513cImMILdRe/+UOIe0Se/D/Z7XCua4AFg5k4Zt3ALMGaC1W3FzlrR2w==" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -963,11 +968,6 @@ "asn1.js": "^5.2.0" } }, - "jwt-decode": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.0.0.tgz", - "integrity": "sha512-RBQv2MTm3FNKQkdzhEyQwh5MbdNgMa+FyIJIK5RMWEn6hRgRHr7j55cRxGhRe6vGJDElyi6f6u/yfkP7AoXddA==" - }, "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", diff --git a/package.json b/package.json index e921a3e..bee4cbf 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "express": "^4.17.1", "flags": "^0.1.3", "irc": "^0.5.2", + "is-port-available": "^0.1.5", "jose": "^1.15.1", "mysql": "^2.17.1", "node-media-server": "^2.2.4", diff --git a/src/cluster.ts b/src/cluster.ts index 72f4fa2..4a5181e 100644 --- a/src/cluster.ts +++ b/src/cluster.ts @@ -8,6 +8,7 @@ import * as strf from "strftime"; import * as ctx from '../node_modules/node-media-server/node_core_ctx'; import * as db from "./database"; import {config} from "./config"; +import * as isPortAvailable from "is-port-available"; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); const { exec, execFile } = require('child_process'); @@ -64,11 +65,14 @@ if (cluster.isMaster) { rtmp: Object.assign({port: 1936}, config['rtmp']) }; + //find a unique port to listen on + getPort().then((wPort) => { + // creating the rtmp server var serv = net.createServer((socket) => { let session = new NodeRtmpSession(rtmpcfg, socket); session.run(); - }).listen(1936); + }).listen(wPort); logger.setLogType(0); // RTMP Server Logic @@ -99,7 +103,7 @@ if (cluster.isMaster) { mkdir(config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username, { recursive : true }, ()=>{;}); while(true){ if(session.audioCodec !== 0 && session.videoCodec !== 0){ - transCommand(results[0].username, key).then((r) => { + transCommand(results[0].username, key, wPort).then((r) => { execFile(config['media']['ffmpeg'], r, {maxBuffer: Infinity}, (err, stdout, stderr) => { /*console.log(err); console.log(stdout); @@ -114,7 +118,7 @@ if (cluster.isMaster) { console.log(`[RTMP Cluster WORKER ${process.pid}] Initiating recording for stream: ${id}`); mkdir(config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username, { recursive : true }, (err) => { if (err) throw err; - execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username+'/'+strf('%d%b%Y-%H%M')+'.mp4'], { + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+wPort+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', config['http']['directory']+'/'+config['media']['publicEndpoint']+'/'+results[0].username+'/'+strf('%d%b%Y-%H%M')+'.mp4'], { detached : true, stdio : 'inherit', maxBuffer: Infinity @@ -130,7 +134,7 @@ if (cluster.isMaster) { db.query('SELECT twitch_key,enabled from twitch_mirror where username='+db.raw.escape(results[0].username)+' limit 1').then(async (tm) => { if(!tm[0]['enabled'] || !config['twitch_mirror']['enabled'] || !config['twitch_mirror']['ingest']) return; console.log('[NodeMediaServer] Mirroring to twitch for stream:',id) - execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { + execFile(config['media']['ffmpeg'], ['-loglevel', 'fatal', '-i', 'rtmp://127.0.0.1:'+wPort+'/'+config['media']['privateEndpoint']+'/'+key, '-vcodec', 'copy', '-acodec', 'copy', '-f', 'flv', config['twitch_mirror']['ingest']+tm[0]['twitch_key']], { detached: true, stdio : 'inherit', maxBuffer: Infinity @@ -206,6 +210,8 @@ if (cluster.isMaster) { } }); console.log(`[RTMP Cluster WORKER ${process.pid}] Worker Ready.`); + + }); } function newRTMPListener(eventName, listener) { @@ -216,10 +222,23 @@ function getRTMPSession(id) { return ctx.sessions.get(id); } -async function transCommand(user: string, key: string): Promise{ +async function getPort(): Promise{ + let port = 1936+process.pid; + while(true){ + let i=0; + if(await isPortAvailable(port+i)){ + port += i; + break; + } + i++; + } + return port; +} + +async function transCommand(user: string, key: string, wPort): Promise{ let args: string[] = ['-loglevel', 'fatal', '-y']; if(config['transcode']['inputflags'] !== null && config['transcode']['inputflags'] !== "") args = args.concat(config['transcode']['inputflags'].split(" ")); - args = args.concat(['-i', 'rtmp://127.0.0.1:'+config['rtmp']['port']+'/'+config['media']['privateEndpoint']+'/'+key, '-movflags', '+faststart']); + args = args.concat(['-i', 'rtmp://127.0.0.1:'+wPort+'/'+config['media']['privateEndpoint']+'/'+key, '-movflags', '+faststart']); if(config['transcode']['adaptive']===true && config['transcode']['variants'] > 1) { for(let i=0;i Date: Sat, 17 Oct 2020 02:17:10 -0500 Subject: [PATCH 31/31] Add configurability to cluster option --- install/config.example.yml | 3 +++ src/index.ts | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/install/config.example.yml b/install/config.example.yml index be1a5a0..8925c03 100644 --- a/install/config.example.yml +++ b/install/config.example.yml @@ -9,6 +9,9 @@ media: ffmpeg: '' rtmp: + # enable cluster mode this will pretty much entirely + # break the ability to play rtmp for clients + cluster: false port: 1935 http: diff --git a/src/index.ts b/src/index.ts index 2573948..d2c3f15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,12 +4,13 @@ import {init as initHTTP} from "./http"; import {init as clean} from "./cleanup"; import {init as initChat} from "./chat"; import { config } from "./config"; +import { execFile } from "child_process"; async function run() { await initDB(); await clean(); await initHTTP(); - await initRTMP(); + config['rtmp']['cluster'] ? execFile(process.cwd()+'/node_modules/.bin/ts-node' [process.cwd()+'src/cluster.ts']) : await initRTMP(); await initChat(); console.log(`Satyr v${config['satyr']['version']} ready`); }