feat: initiate db connection only once

db is now initiated via utilsController, and gets re-used by main
lolisafe.js script and the other controllers

this should now make sure we only have 1 active db connection pool ever

this is a pretty big refactor due to how we used to reference db,
but it should functionally be identical
This commit is contained in:
Bobby 2022-06-04 04:21:56 +07:00
parent 6768fa46f6
commit 8799189a04
No known key found for this signature in database
GPG Key ID: 941839794CBF5A09
10 changed files with 80 additions and 92 deletions

View File

@ -12,7 +12,6 @@ const ClientError = require('./utils/ClientError')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const logger = require('./../logger')
const db = require('knex')(config.database)
const self = {
// Don't forget to update max length of text inputs in
@ -56,7 +55,7 @@ self.getUniqueRandomName = async () => {
// Put token on-hold (wait for it to be inserted to DB)
self.onHold.add(identifier)
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where('identifier', identifier)
.select('id')
.first()
@ -91,7 +90,7 @@ self.list = async (req, res, next) => {
}
// Query albums count for pagination
const count = await db.table('albums')
const count = await utils.db.table('albums')
.where(filter)
.count('id as count')
.then(rows => rows[0].count)
@ -101,7 +100,7 @@ self.list = async (req, res, next) => {
let albums
if (simple) {
albums = await db.table('albums')
albums = await utils.db.table('albums')
.where(filter)
.select(fields)
@ -114,7 +113,7 @@ self.list = async (req, res, next) => {
fields.push('identifier', 'enabled', 'timestamp', 'editedAt', 'zipGeneratedAt', 'download', 'public', 'description')
if (all) fields.push('userid')
albums = await db.table('albums')
albums = await utils.db.table('albums')
.where(filter)
.limit(25)
.offset(25 * offset)
@ -149,7 +148,7 @@ self.list = async (req, res, next) => {
await Promise.all(albums.map(album => getAlbumZipSize(album)))
const uploads = await db.table('files')
const uploads = await utils.db.table('files')
.whereIn('albumid', Object.keys(albumids))
.select('albumid', 'size')
@ -174,7 +173,7 @@ self.list = async (req, res, next) => {
if (!userids.length) return res.json({ success: true, albums, count, homeDomain })
// Query usernames of user IDs from currently selected files
const usersTable = await db.table('users')
const usersTable = await utils.db.table('users')
.whereIn('id', userids)
.select('id', 'username')
@ -199,7 +198,7 @@ self.create = async (req, res, next) => {
if (!name) throw new ClientError('No album name specified.')
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where({
name,
enabled: 1,
@ -211,7 +210,7 @@ self.create = async (req, res, next) => {
const identifier = await self.getUniqueRandomName()
const ids = await db.table('albums').insert({
const ids = await utils.db.table('albums').insert({
name,
enabled: 1,
userid: user.id,
@ -263,7 +262,7 @@ self.disable = async (req, res, next) => {
}
}
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where(filter)
.first()
@ -272,7 +271,7 @@ self.disable = async (req, res, next) => {
}
if (purge) {
const files = await db.table('files')
const files = await utils.db.table('files')
.where({
albumid: id,
userid: album.userid
@ -287,12 +286,12 @@ self.disable = async (req, res, next) => {
}
if (del) {
await db.table('albums')
await utils.db.table('albums')
.where(filter)
.first()
.del()
} else {
await db.table('albums')
await utils.db.table('albums')
.where(filter)
.first()
.update('enabled', 0)
@ -338,7 +337,7 @@ self.edit = async (req, res, next) => {
}
}
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where(filter)
.first()
@ -350,7 +349,7 @@ self.edit = async (req, res, next) => {
? Boolean(req.body.enabled)
: null
const nameInUse = await db.table('albums')
const nameInUse = await utils.db.table('albums')
.where({
name,
enabled: 1,
@ -385,7 +384,7 @@ self.edit = async (req, res, next) => {
update.identifier = await self.getUniqueRandomName()
}
await db.table('albums')
await utils.db.table('albums')
.where(filter)
.update(update)
utils.invalidateAlbumsCache([id])
@ -429,7 +428,7 @@ self.get = async (req, res, next) => {
throw new ClientError('No identifier provided.')
}
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where({
identifier,
enabled: 1
@ -441,7 +440,7 @@ self.get = async (req, res, next) => {
}
const title = album.name
const files = await db.table('files')
const files = await utils.db.table('files')
.select('name')
.where('albumid', album.id)
.orderBy('id', 'desc')
@ -486,7 +485,7 @@ self.generateZip = async (req, res, next) => {
throw new ClientError('ZIP generation disabled.', { statusCode: 403 })
}
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where({
identifier,
enabled: 1
@ -530,7 +529,7 @@ self.generateZip = async (req, res, next) => {
logger.log(`Starting zip task for album: ${identifier}.`)
const files = await db.table('files')
const files = await utils.db.table('files')
.select('name', 'size')
.where('albumid', album.id)
if (files.length === 0) {
@ -574,7 +573,7 @@ self.generateZip = async (req, res, next) => {
logger.log(`Finished zip task for album: ${identifier} (success).`)
await db.table('albums')
await utils.db.table('albums')
.where('id', album.id)
.update('zipGeneratedAt', Math.floor(Date.now() / 1000))
utils.invalidateStatsCache('albums')
@ -641,7 +640,7 @@ self.addFiles = async (req, res, next) => {
failed = []
albumids = []
if (albumid !== null) {
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where('id', albumid)
.where(function () {
if (user.username !== 'root') {
@ -657,13 +656,13 @@ self.addFiles = async (req, res, next) => {
albumids.push(albumid)
}
const files = await db.table('files')
const files = await utils.db.table('files')
.whereIn('id', ids)
.where('userid', user.id)
failed = ids.filter(id => !files.find(file => file.id === id))
await db.table('files')
await utils.db.table('files')
.whereIn('id', files.map(file => file.id))
.update('albumid', albumid)
utils.invalidateStatsCache('albums')
@ -674,7 +673,7 @@ self.addFiles = async (req, res, next) => {
}
})
await db.table('albums')
await utils.db.table('albums')
.whereIn('id', albumids)
.update('editedAt', Math.floor(Date.now() / 1000))
utils.invalidateAlbumsCache(albumids)

View File

@ -9,7 +9,6 @@ const apiErrorsHandler = require('./handlers/apiErrorsHandler.js')
const ClientError = require('./utils/ClientError')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const db = require('knex')(config.database)
// Don't forget to update min/max length of text inputs in auth.njk
// when changing these values.
@ -44,7 +43,7 @@ self.verify = async (req, res, next) => {
: ''
if (!password) throw new ClientError('No password provided.')
const user = await db.table('users')
const user = await utils.db.table('users')
.where('username', username)
.first()
@ -85,7 +84,7 @@ self.register = async (req, res, next) => {
throw new ClientError(`Password must have ${self.pass.min}-${self.pass.max} characters.`)
}
const user = await db.table('users')
const user = await utils.db.table('users')
.where('username', username)
.first()
@ -98,7 +97,7 @@ self.register = async (req, res, next) => {
throw new ServerError('Failed to allocate a unique token. Try again?')
}
await db.table('users')
await utils.db.table('users')
.insert({
username,
password: hash,
@ -129,7 +128,7 @@ self.changePassword = async (req, res, next) => {
const hash = await bcrypt.hash(password, saltRounds)
await db.table('users')
await utils.db.table('users')
.where('id', user.id)
.update('password', hash)
@ -184,7 +183,7 @@ self.createUser = async (req, res, next) => {
}
}
const exists = await db.table('users')
const exists = await utils.db.table('users')
.where('username', username)
.first()
@ -197,7 +196,7 @@ self.createUser = async (req, res, next) => {
throw new ServerError('Failed to allocate a unique token. Try again?')
}
await db.table('users')
await utils.db.table('users')
.insert({
username,
password: hash,
@ -225,7 +224,7 @@ self.editUser = async (req, res, next) => {
const id = parseInt(req.body.id)
if (isNaN(id)) throw new ClientError('No user specified.')
const target = await db.table('users')
const target = await utils.db.table('users')
.where('id', id)
.first()
self.assertPermission(user, target)
@ -256,7 +255,7 @@ self.editUser = async (req, res, next) => {
update.password = await bcrypt.hash(password, saltRounds)
}
await db.table('users')
await utils.db.table('users')
.where('id', id)
.update(update)
utils.invalidateStatsCache('users')
@ -285,12 +284,12 @@ self.deleteUser = async (req, res, next) => {
const purge = req.body.purge
if (isNaN(id)) throw new ClientError('No user specified.')
const target = await db.table('users')
const target = await utils.db.table('users')
.where('id', id)
.first()
self.assertPermission(user, target)
const files = await db.table('files')
const files = await utils.db.table('files')
.where('userid', id)
.select('id')
@ -302,20 +301,20 @@ self.deleteUser = async (req, res, next) => {
utils.invalidateStatsCache('uploads')
} else {
// Clear out userid attribute from the files
await db.table('files')
await utils.db.table('files')
.whereIn('id', fileids)
.update('userid', null)
}
}
const albums = await db.table('albums')
const albums = await utils.db.table('albums')
.where('userid', id)
.where('enabled', 1)
.select('id', 'identifier')
if (albums.length) {
const albumids = albums.map(album => album.id)
await db.table('albums')
await utils.db.table('albums')
.whereIn('id', albumids)
.del()
utils.invalidateAlbumsCache(albumids)
@ -331,7 +330,7 @@ self.deleteUser = async (req, res, next) => {
}))
}
await db.table('users')
await utils.db.table('users')
.where('id', id)
.del()
utils.invalidateStatsCache('users')
@ -353,7 +352,7 @@ self.listUsers = async (req, res, next) => {
const isadmin = perms.is(user, 'admin')
if (!isadmin) throw new ClientError('', { statusCode: 403 })
const count = await db.table('users')
const count = await utils.db.table('users')
.count('id as count')
.then(rows => rows[0].count)
if (!count) return res.json({ success: true, users: [], count })
@ -362,7 +361,7 @@ self.listUsers = async (req, res, next) => {
if (isNaN(offset)) offset = 0
else if (offset < 0) offset = Math.max(0, Math.ceil(count / 25) + offset)
const users = await db.table('users')
const users = await utils.db.table('users')
.limit(25)
.offset(25 * offset)
.select('id', 'username', 'enabled', 'timestamp', 'permission', 'registration')
@ -376,7 +375,7 @@ self.listUsers = async (req, res, next) => {
pointers[user.id] = user
}
const uploads = await db.table('files')
const uploads = await utils.db.table('files')
.whereIn('userid', Object.keys(pointers))
.select('userid', 'size')

View File

@ -4,8 +4,6 @@ const utils = require('./utilsController')
const apiErrorsHandler = require('./handlers/apiErrorsHandler')
const ClientError = require('./utils/ClientError')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const db = require('knex')(config.database)
const self = {
tokenLength: 64,
@ -21,7 +19,7 @@ self.generateUniqueToken = async () => {
// Put token on-hold (wait for it to be inserted to DB)
self.onHold.add(token)
const user = await db.table('users')
const user = await utils.db.table('users')
.where('token', token)
.select('id')
.first()
@ -44,7 +42,7 @@ self.verify = async (req, res, next) => {
if (!token) throw new ClientError('No token provided.', { statusCode: 403 })
const user = await db.table('users')
const user = await utils.db.table('users')
.where('token', token)
.select('username', 'permission')
.first()
@ -96,7 +94,7 @@ self.change = async (req, res, next) => {
throw new ServerError('Failed to allocate a unique token. Try again?')
}
await db.table('users')
await utils.db.table('users')
.where('token', user.token)
.update({
token: newToken,

View File

@ -14,7 +14,6 @@ const multerStorage = require('./utils/multerStorage')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const logger = require('./../logger')
const db = require('knex')(config.database)
const self = {
onHold: new Set(),
@ -213,7 +212,7 @@ self.getUniqueRandomName = async (length, extension) => {
// Put token on-hold (wait for it to be inserted to DB)
self.onHold.add(identifier)
const file = await db.table('files')
const file = await utils.db.table('files')
.whereRaw('?? like ?', ['name', `${identifier}.%`])
.select('id')
.first()
@ -827,7 +826,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
await Promise.all(infoMap.map(async info => {
// Check if the file exists by checking its hash and size
const dbFile = await db.table('files')
const dbFile = await utils.db.table('files')
.where(function () {
if (user === undefined) {
this.whereNull('userid')
@ -892,7 +891,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
if (files.length) {
let authorizedIds = []
if (albumids.length) {
authorizedIds = await db.table('albums')
authorizedIds = await utils.db.table('albums')
.where({ userid: user.id })
.whereIn('id', albumids)
.select('id')
@ -907,7 +906,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
}
// Insert new files to DB
await db.table('files').insert(files)
await utils.db.table('files').insert(files)
utils.invalidateStatsCache('uploads')
if (config.uploads.queryDbForFileCollisions) {
@ -920,7 +919,7 @@ self.storeFilesToDb = async (req, res, user, infoMap) => {
// Update albums' timestamp
if (authorizedIds.length) {
await db.table('albums')
await utils.db.table('albums')
.whereIn('id', authorizedIds)
.update('editedAt', Math.floor(Date.now() / 1000))
utils.invalidateAlbumsCache(authorizedIds)
@ -1253,7 +1252,7 @@ self.list = async (req, res, next) => {
usernames.push(...filterObj.queries.exclude.user)
}
const uploaders = await db.table('users')
const uploaders = await utils.db.table('users')
.whereIn('username', usernames)
.select('id', 'username')
@ -1490,7 +1489,7 @@ self.list = async (req, res, next) => {
}
// Query uploads count for pagination
const count = await db.table('files')
const count = await utils.db.table('files')
.where(filter)
.count('id as count')
.then(rows => rows[0].count)
@ -1516,16 +1515,16 @@ self.list = async (req, res, next) => {
orderByRaw = sortObj.parsed.map(sort => {
// Use Knex.raw() to sanitize user inputs
if (sort.cast) {
return db.raw(`cast (?? as ${sort.cast}) ${sort.order} ${sort.clause}`.trim(), sort.column)
return utils.db.raw(`cast (?? as ${sort.cast}) ${sort.order} ${sort.clause}`.trim(), sort.column)
} else {
return db.raw(`?? ${sort.order} ${sort.clause}`.trim(), sort.column)
return utils.db.raw(`?? ${sort.order} ${sort.clause}`.trim(), sort.column)
}
}).join(', ')
} else {
orderByRaw = '`id` desc'
}
const files = await db.table('files')
const files = await utils.db.table('files')
.where(filter)
.orderByRaw(orderByRaw)
.limit(25)
@ -1549,7 +1548,7 @@ self.list = async (req, res, next) => {
.filter((v, i, a) => {
return v !== null && v !== undefined && v !== '' && a.indexOf(v) === i
})
albums = await db.table('albums')
albums = await utils.db.table('albums')
.whereIn('id', albumids)
.where('enabled', 1)
.select('id', 'name')
@ -1579,7 +1578,7 @@ self.list = async (req, res, next) => {
if (!userids.length) return res.json({ success: true, files, count, albums, basedomain })
// Query usernames of user IDs from currently selected files
usersTable = await db.table('users')
usersTable = await utils.db.table('users')
.whereIn('id', userids)
.select('id', 'username')
}

View File

@ -2,6 +2,7 @@ const { promisify } = require('util')
const fetch = require('node-fetch')
const ffmpeg = require('fluent-ffmpeg')
const MarkdownIt = require('markdown-it')
const knex = require('knex')
const path = require('path')
const sharp = require('sharp')
const si = require('systeminformation')
@ -12,9 +13,9 @@ const ClientError = require('./utils/ClientError')
const ServerError = require('./utils/ServerError')
const config = require('./../config')
const logger = require('./../logger')
const db = require('knex')(config.database)
const self = {
db: knex(config.database),
scan: {
instance: null,
version: null,
@ -296,7 +297,7 @@ self.assertUser = async (token, fields) => {
_fields.push(...fields)
}
const user = await db.table('users')
const user = await self.db.table('users')
.where('token', token)
.select(_fields)
.first()
@ -539,7 +540,7 @@ self.bulkDeleteFromDb = async (field, values, user) => {
const albumids = []
await Promise.all(chunks.map(async chunk => {
const files = await db.table('files')
const files = await self.db.table('files')
.whereIn(field, chunk)
.where(function () {
if (!ismoderator) {
@ -566,7 +567,7 @@ self.bulkDeleteFromDb = async (field, values, user) => {
if (!unlinked.length) return
// Delete all unlinked files from db
await db.table('files')
await self.db.table('files')
.whereIn('id', unlinked.map(file => file.id))
.del()
self.invalidateStatsCache('uploads')
@ -593,7 +594,7 @@ self.bulkDeleteFromDb = async (field, values, user) => {
if (unlinkeds.length) {
// Update albums if necessary, but do not wait
if (albumids.length) {
db.table('albums')
self.db.table('albums')
.whereIn('id', albumids)
.update('editedAt', Math.floor(Date.now() / 1000))
.catch(logger.error)
@ -735,7 +736,7 @@ self.bulkDeleteExpired = async (dryrun, verbose) => {
const sudo = { username: 'root' }
const result = {}
result.expired = await db.table('files')
result.expired = await self.db.table('files')
.where('expirydate', '<=', timestamp)
.select(fields)
@ -904,14 +905,14 @@ self.stats = async (req, res, next) => {
}
const getTotalCountAndSize = async () => {
const uploads = await db.table('files')
const uploads = await self.db.table('files')
.select('size')
stats[data.title].Total = uploads.length
stats[data.title]['Size in DB'].value = uploads.reduce((acc, upload) => acc + parseInt(upload.size), 0)
}
const getImagesCount = async () => {
stats[data.title].Images = await db.table('files')
stats[data.title].Images = await self.db.table('files')
.where(function () {
for (const ext of self.imageExts) {
this.orWhere('name', 'like', `%${ext}`)
@ -922,7 +923,7 @@ self.stats = async (req, res, next) => {
}
const getVideosCount = async () => {
stats[data.title].Videos = await db.table('files')
stats[data.title].Videos = await self.db.table('files')
.where(function () {
for (const ext of self.videoExts) {
this.orWhere('name', 'like', `%${ext}`)
@ -933,7 +934,7 @@ self.stats = async (req, res, next) => {
}
const getAudiosCount = async () => {
stats[data.title].Audios = await db.table('files')
stats[data.title].Audios = await self.db.table('files')
.where(function () {
for (const ext of self.audioExts) {
this.orWhere('name', 'like', `%${ext}`)
@ -944,7 +945,7 @@ self.stats = async (req, res, next) => {
}
const getOthersCount = async () => {
stats[data.title].Temporary = await db.table('files')
stats[data.title].Temporary = await self.db.table('files')
.whereNotNull('expirydate')
.count('id as count')
.then(rows => rows[0].count)
@ -991,7 +992,7 @@ self.stats = async (req, res, next) => {
stats[data.title][p] = 0
})
const users = await db.table('users')
const users = await self.db.table('users')
stats[data.title].Total = users.length
for (const user of users) {
if (user.enabled === false || user.enabled === 0) {
@ -1033,7 +1034,7 @@ self.stats = async (req, res, next) => {
'ZIP Generated': 0
}
const albums = await db.table('albums')
const albums = await self.db.table('albums')
stats[data.title].Total = albums.length
const activeAlbums = []
@ -1051,7 +1052,7 @@ self.stats = async (req, res, next) => {
stats[data.title]['ZIP Generated'] = files.length
}).catch(() => {})
stats[data.title]['Files in albums'] = await db.table('files')
stats[data.title]['Files in albums'] = await self.db.table('files')
.whereIn('albumid', activeAlbums)
.count('id as count')
.then(rows => rows[0].count)

View File

@ -48,7 +48,6 @@ const api = require('./routes/api')
const nojs = require('./routes/nojs')
const player = require('./routes/player')
const db = require('knex')(config.database)
const isDevMode = process.env.NODE_ENV === 'development'
// Helmet security headers
@ -144,7 +143,7 @@ const initServeStaticUploads = (opts = {}) => {
const relpath = path.replace(paths.uploads, '')
if (relpath.indexOf('/', 1) === -1 && req.method === 'GET') {
const name = relpath.substring(1)
const file = await db.table('files')
const file = await utils.db.table('files')
.where('name', name)
.select('original')
.first()
@ -250,7 +249,7 @@ safe.use('/api', api)
;(async () => {
try {
// Init database
await require('./database/db.js')(db)
await require('./database/db.js')(utils.db)
// Verify paths, create missing ones, clean up temp ones
await paths.init()
@ -335,7 +334,7 @@ safe.use('/api', api)
// Cache file identifiers
if (config.uploads.cacheFileIdentifiers) {
utils.idSet = await db.table('files')
utils.idSet = await utils.db.table('files')
.select('name')
.then(rows => {
return new Set(rows.map(row => row.name.split('.')[0]))

View File

@ -3,7 +3,6 @@ const path = require('path')
const paths = require('./../controllers/pathsController')
const utils = require('./../controllers/utilsController')
const config = require('./../config')
const db = require('knex')(config.database)
routes.get('/a/:identifier', async (req, res, next) => {
const identifier = req.params.identifier
@ -11,7 +10,7 @@ routes.get('/a/:identifier', async (req, res, next) => {
res.status(404).sendFile(path.join(paths.errorRoot, config.errorPages[404]))
}
const album = await db.table('albums')
const album = await utils.db.table('albums')
.where({
identifier,
enabled: 1
@ -51,7 +50,7 @@ routes.get('/a/:identifier', async (req, res, next) => {
utils.albumsCache[cacheid].generating = true
}
const files = await db.table('files')
const files = await utils.db.table('files')
.select('name', 'size')
.where('albumid', album.id)
.orderBy('id', 'desc')

View File

@ -1,8 +1,6 @@
const path = require('path')
const paths = require('../controllers/pathsController')
const utils = require('../controllers/utilsController')
const config = require('./../config')
const db = require('knex')(config.database)
const self = {
getFiles: async directory => {
@ -43,7 +41,7 @@ const self = {
const uploads = await self.getFiles(paths.uploads)
console.log(`Uploads: ${uploads.length}`)
const uploadsDb = await db.table('files')
const uploadsDb = await utils.db.table('files')
.select('name')
.then(rows => rows.map(row => row.name))
console.log(`- In DB: ${uploadsDb.length}`)

View File

@ -3,8 +3,6 @@ const fs = require('fs')
const path = require('path')
const paths = require('../controllers/pathsController')
const utils = require('../controllers/utilsController')
const config = require('./../config')
const db = require('knex')(config.database)
;(async () => {
const location = process.argv[1].replace(process.cwd() + '/', '')
@ -33,7 +31,7 @@ const db = require('knex')(config.database)
console.log('Querying uploads\u2026')
const hrstart = process.hrtime()
const uploads = await db.table('files')
const uploads = await utils.db.table('files')
.select('id', 'name', 'hash')
console.log(`Uploads : ${uploads.length}`)
@ -56,7 +54,7 @@ const db = require('knex')(config.database)
const hash = source.toString('hex')
if (verbose) console.log(`${upload.name}: ${hash}`)
if (!dryrun && upload.hash !== hash) {
await db.table('files')
await utils.db.table('files')
.update('hash', hash)
.where('id', upload.id)
}

View File

@ -1,8 +1,6 @@
const path = require('path')
const paths = require('../controllers/pathsController')
const utils = require('../controllers/utilsController')
const config = require('./../config')
const db = require('knex')(config.database)
const self = {
mode: null,
@ -53,7 +51,7 @@ const self = {
console.log('Looking through existing thumbnails\u2026')
const hrstart = process.hrtime()
const uploads = await db.table('files')
const uploads = await utils.db.table('files')
.select('id', 'name')
const thumbs = await self.getFiles(paths.thumbs)
.then(thumbs => thumbs.map(thumb => {