Added scripts/rebuild-hashes.js

Closes #345

controllers/utilsController.js

@@ -176,6 +176,36 @@ self.stripIndents = string => {
   return result
 }
 
+self.parallelLimit = (promiseFactories, limit, progressCallback) => {
+  // https://stackoverflow.com/a/40377750
+  // CC BY-SA 3.0
+  const hasProgressCallback = typeof progressCallback === 'function'
+  const total = promiseFactories.length
+  const result = []
+  let cnt = 0
+
+  function chain (promiseFactories) {
+    if (!promiseFactories.length) return
+    const i = cnt++ // preserve order in result
+    if (hasProgressCallback) {
+      progressCallback({ done: cnt, total })
+    }
+    return promiseFactories.shift().then((res) => {
+      result[i] = res // save result
+      return chain(promiseFactories) // append next promise
+    })
+  }
+
+  const arrChains = []
+  while (limit-- > 0 && promiseFactories.length > 0) {
+    // create "limit" chains which run in parallel
+    arrChains.push(chain(promiseFactories))
+  }
+
+  // return when all arrChains are finished
+  return Promise.all(arrChains).then(() => result)
+}
+
 self.authorize = async (req, res) => {
   // TODO: Improve usage of this function by the other APIs
   const token = req.headers.token

scripts/bump-versions.js

@@ -61,7 +61,7 @@ const self = {
       4: Renders from /public/render/* directories (to be used with /src/js/misc/render.js).
       5: Fontello font files.
       a: Shortcut to update all types.
-    `))
+    `).trim())
   }
 
   const file = path.resolve('./src/versions.json')

scripts/cf-purge.js

@@ -13,7 +13,7 @@ const utils = require('./../controllers/utilsController')
 
       filename:
       Upload names separated by space (will automatically include their thumbs if available).
-    `))
+    `).trim())
   }
 
   const results = await utils.purgeCloudflareCache(args, true, true)

scripts/clean-up.js

@@ -34,7 +34,7 @@ self.getFiles = async directory => {
       mode:
       0 = Only list names of files that are not in the database.
       1 = Clean up the files.
-    `))
+    `).trim())
   }
 
   self.mode = parseInt(args[0]) || 0

scripts/delete-expired.js

@@ -19,7 +19,7 @@ const self = {
       0 = Only list names of the expired files.
       1 = Delete expired files (output file names).
       2 = Delete expired files (no output).
-    `))
+    `).trim())
   }
 
   self.mode = parseInt(args[0]) || 0

scripts/rebuild-hashes.js

@@ -0,0 +1,83 @@
+const blake3 = require('blake3')
+const fs = require('fs')
+const path = require('path')
+const paths = require('../controllers/pathsController')
+const utils = require('../controllers/utilsController')
+const config = require('./../config')
+const db = require('knex')(config.database)
+
+const self = {
+  mode: null
+}
+
+;(async () => {
+  const location = process.argv[1].replace(process.cwd() + '/', '')
+  const args = process.argv.slice(2)
+
+  self.mode = parseInt(args[0])
+
+  if (![0, 1, 2].includes(self.mode) ||
+    args.includes('--help') ||
+    args.includes('-h')) {
+    return console.log(utils.stripIndents(`
+      Rebuild file hashes.
+
+      Usage  :
+      node ${location} <mode=0|1|2> [parallel]
+
+      mode:
+      0 = Dry run (recalculate hashes, print them, but do NOT store to DB).
+      1 = Recalculate hashes, print them, and store to DB.
+      2 = Quiet (recalculate hashes and store to DB).
+
+      parallel:
+      Amount of uploads to hash in parallel (not to be confused with multi-threading).
+    `).trim())
+  }
+
+  const dryrun = self.mode === 0
+  const quiet = self.mode === 2
+  const parallel = Math.min(Math.max(parseInt(args[1]), 0), 32) || 8
+
+  const hrstart = process.hrtime()
+  const uploads = await db.table('files')
+    .select('id', 'name', 'hash')
+  console.log(`Uploads : ${uploads.length}`)
+  console.log(`Parallel: ${parallel}`)
+
+  let lastProgressOut
+  await utils.parallelLimit(uploads.map(upload => {
+    return new Promise((resolve, reject) => {
+      fs.createReadStream(path.join(paths.uploads, upload.name))
+        .on('error', () => reject)
+        .pipe(blake3.createHash())
+        .on('data', async hasher => {
+          const hash = hasher.toString('hex')
+          if (!quiet) console.log(`${upload.name}: ${hash}`)
+          if (!dryrun && upload.hash !== hash) {
+            await db.table('files')
+              .update('hash', hash)
+              .where('id', upload.id)
+          }
+          resolve()
+        })
+    })
+  }), parallel, progress => {
+    const now = Date.now()
+    if (!lastProgressOut || (now - lastProgressOut >= 1000) || progress.done === progress.total) {
+      console.log(`Progress: ${progress.done}/${progress.total}`)
+      lastProgressOut = now
+    }
+  })
+
+  const hrend = process.hrtime(hrstart)
+  console.log(`Done in : ${(hrend[0] + (hrend[1] / 1e9)).toFixed(4)}s`)
+  if (dryrun) {
+    console.log('INFO: This was a dry run. DB had not been modified.')
+  }
+})()
+  .then(() => process.exit(0))
+  .catch(error => {
+    console.error(error)
+    process.exit(1)
+  })

scripts/thumbs.js

@@ -50,7 +50,7 @@ self.getFiles = async directory => {
       force  : 0 = no force (default), 1 = overwrite existing thumbnails
       verbose: 0 = only print missing thumbs (default), 1 = print all
       cfcache: 0 = do not clear cloudflare cache (default), 1 = clear cloudflare cache
-    `))
+    `).trim())
   }
 
   console.log('Looking through existing thumbnails\u2026')

src/js/misc/newsfeed.js

@@ -29,6 +29,7 @@ newsfeed.simpleParseDate = string => {
 
 newsfeed.formatRelativeDate = delta => {
   // https://stackoverflow.com/a/7641812
+  // CC BY-SA 3.0
   const minute = 60
   const hour = minute * 60
   const day = hour * 24