diff --git a/.github/README.md b/.github/README.md index 8266efe..4e5a392 100644 --- a/.github/README.md +++ b/.github/README.md @@ -2,16 +2,16 @@ - Invite only registration or disable registration completely. - Upload files, images, and videos. -- Manage users, roles. +- Manage users role. - Limit user quota. - Discord Embed customizer. -- Download and delete files. -- File preview. +- Download and delete files from the dashboard. - Downloadable upload config for ShareX, Flameshot. - A view page for each file. - S3 support (AWS, DigitalOcean, etc.). - Easy installation with docker. - Easy to use admin panel. +- Email verification. # Installing @@ -63,6 +63,7 @@ docker compose pull && docker compose up -d - `pm2` globally installed - `yarn` globally installed - `caddy` installed +- `ffmpeg` installed ### Backend Installation @@ -125,6 +126,7 @@ Then go through the installation steps again. ### Domain name and SSL configuration If you don't have caddy installed already +
[Click Here](https://caddyserver.com/docs/install) Copy and paste the following into your terminal: diff --git a/.gitignore b/.gitignore index 412c257..7b6ccf0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -docker-compose.yml \ No newline at end of file +docker-compose.yml +db_data +redis_data diff --git a/api/.dockerignore b/api/.dockerignore index e199007..99373a9 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -12,4 +12,6 @@ /.yarn/* !/.yarn/releases !/.yarn/plugins -/test \ No newline at end of file +/test +.env +.env.example \ No newline at end of file diff --git a/api/Dockerfile.dev b/api/Dockerfile.dev new file mode 100644 index 0000000..a304cb7 --- /dev/null +++ b/api/Dockerfile.dev @@ -0,0 +1,18 @@ +FROM node:lts-buster-slim AS development + +ENV NODE_ENV development + +RUN apt-get update && apt-get install --no-install-recommends -y \ + openssl \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY package.json yarn.lock .yarnrc.yml ./ +COPY .yarn ./.yarn + +RUN yarn install --immutable + +COPY . . + +CMD [ "yarn", "start:dev" ] \ No newline at end of file diff --git a/api/env.d.ts b/api/env.d.ts index 4a2af0e..8d8b04d 100644 --- a/api/env.d.ts +++ b/api/env.d.ts @@ -19,5 +19,8 @@ declare namespace NodeJS { S3_REGION: string; CDN_URL: string; UPLOADER: "local" | "s3"; + // optional env vars + COOKIE_NAME?: string; + UPLOAD_DIR?: string; } } diff --git a/api/package.json b/api/package.json index 39dd035..ec066be 100644 --- a/api/package.json +++ b/api/package.json @@ -37,6 +37,7 @@ "cuid": "^2.1.8", "express-session": "^1.17.3", "fast-folder-size": "^1.7.1", + "ffmpeg-static": "^5.1.0", "hbs": "^4.2.0", "helmet": "^6.0.0", "ioredis": "^5.2.4", diff --git a/api/prisma/migrations/20221229053707_album_cover/migration.sql b/api/prisma/migrations/20221229053707_album_cover/migration.sql new file mode 100644 index 0000000..44e94c0 --- /dev/null +++ b/api/prisma/migrations/20221229053707_album_cover/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "files" ADD COLUMN "cover" TEXT; diff --git a/api/prisma/migrations/20221229055446_remove_cover/migration.sql b/api/prisma/migrations/20221229055446_remove_cover/migration.sql new file mode 100644 index 0000000..f61220a --- /dev/null +++ b/api/prisma/migrations/20221229055446_remove_cover/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - You are about to drop the column `cover` on the `files` table. All the data in the column will be lost. + +*/ +-- AlterTable +ALTER TABLE "files" DROP COLUMN "cover"; diff --git a/api/src/app.module.ts b/api/src/app.module.ts index d056e4e..a8c22e8 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -15,6 +15,7 @@ import { RedisService } from "modules/redis/redis.service"; import { ThrottlerModule } from "@nestjs/throttler"; import { APP_GUARD } from "@nestjs/core"; import { ThrottlerBehindProxyGuard } from "modules/root/root.guard"; +import { S3Service } from "modules/s3/s3.service"; @Module({ imports: [ @@ -32,6 +33,7 @@ import { ThrottlerBehindProxyGuard } from "modules/root/root.guard"; providers: [ RootService, PrismaService, + S3Service, RedisService, { provide: APP_GUARD, diff --git a/api/src/lib/clean.ts b/api/src/lib/clean.ts index f285787..0cad21f 100644 --- a/api/src/lib/clean.ts +++ b/api/src/lib/clean.ts @@ -1,26 +1,67 @@ import { uploadDir } from "./constants"; import { promises as fs } from "fs"; import { CronJob } from "cron"; -import { join } from "path"; +import { Logger } from "@nestjs/common"; +import { exec } from "child_process"; const cleanUp = async () => { - // find files start with tmp_ that are older than 24 hours - const tmpFiles = (await fs.readdir(uploadDir)) - .filter((file) => file.startsWith("tmp_")) - .filter(async (file) => { - const { birthtime } = await fs.stat(join(uploadDir, file)); - return Date.now() - birthtime.getTime() > 1000 * 60 * 60 * 24; - }); - const currentTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; const job = new CronJob( // every 24 hours at 12 AM "0 0 * * *", async () => { - console.log("Cleaning up tmp files..."); - for (const file of tmpFiles) { - await fs.unlink(join(uploadDir, file)); + const startTime = Date.now(); + + const logger = new Logger("CronJob"); + + const isWindows = process.platform === "win32"; + + if (isWindows) { + // check if forfiles.exe exists + try { + await fs.access("C:\\Windows\\System32\\forfiles.exe"); + } catch (error) { + logger.error("forfiles.exe not found, aborting job"); + logger.log(`Finished job in ${Date.now() - startTime}ms`); + return; + } + + const { stdout } = await exec( + `forfiles /p ${uploadDir} /s /m tmp_* /D -1 /c "cmd /c del @path"` + ); + + stdout.on("data", (data) => { + logger.log(data); + }); + + logger.log(`Finished job in ${Date.now() - startTime}ms`); + + return; + } + + const isUnix = + process.platform === "linux" || + process.platform === "darwin" || + process.platform === "freebsd" || + process.platform === "openbsd"; + + if (isUnix) { + const { stdout } = await exec( + `find ${uploadDir} -type f -name "tmp_*" -mtime +1 -exec rm -f {} \\;` + ); + + stdout.on("data", (data) => { + logger.log(data); + }); + + logger.log(`Finished job in ${Date.now() - startTime}ms`); + + return; + } else { + logger.log("OS not supported"); + logger.log(`Finished job in ${Date.now() - startTime}ms`); + return; } }, null, @@ -28,6 +69,12 @@ const cleanUp = async () => { currentTimeZone ); + const supportedOS = ["win32", "linux", "darwin", "freebsd", "openbsd"]; + + if (!supportedOS.includes(process.platform)) { + return; + } + job.start(); }; diff --git a/api/src/lib/constants.ts b/api/src/lib/constants.ts index f0db064..d405e1a 100644 --- a/api/src/lib/constants.ts +++ b/api/src/lib/constants.ts @@ -5,14 +5,11 @@ export enum ROUTES { USERS = "users", UPLOAD = "upload", DELETE = "delete", - STATISTICS = "statistics", } export const rootDir = join(__dirname, "..", ".."); -export const uploadDir = join(rootDir, "uploads"); -export const thumbnailDir = join(rootDir, "public"); +export const uploadDir = process.env.UPLOAD_DIR ?? join(rootDir, "uploads"); export const logsDir = join(rootDir, "logs"); -export const tmpDir = join(rootDir, "tmp"); export const COOKIE_NAME = process.env.COOKIE_NAME ?? "auth"; export const INVITE_PREFIX = "invite:"; export const FORGOT_PASSWORD_PREFIX = "forgot-password:"; diff --git a/api/src/lib/types.ts b/api/src/lib/types.ts index bf537bd..c4eaffd 100644 --- a/api/src/lib/types.ts +++ b/api/src/lib/types.ts @@ -16,6 +16,7 @@ export interface UserResponse { export interface findUserOptions { byId?: boolean; withPassword?: boolean; + withFiles?: boolean; totalUsed?: boolean; } diff --git a/api/src/main.ts b/api/src/main.ts index dc05eed..3e94071 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -53,7 +53,9 @@ async function bootstrap() { resave: false, saveUninitialized: false, cookie: { - secure: process.env.NODE_ENV === "production", + secure: + process.env.NODE_ENV === "production" && + process.env.USE_SSL === "true", httpOnly: true, maxAge: 1000 * 60 * 60 * 24 * 7, // 7 days sameSite: "lax", diff --git a/api/src/modules/root/root.service.ts b/api/src/modules/root/root.service.ts index 4a83b06..7f2a24b 100644 --- a/api/src/modules/root/root.service.ts +++ b/api/src/modules/root/root.service.ts @@ -8,7 +8,7 @@ import { Request, Response } from "express"; import fastFolderSize from "fast-folder-size"; import { createReadStream, existsSync } from "fs"; import { stat } from "fs/promises"; -import { thumbnailDir, uploadDir } from "lib/constants"; +import { uploadDir } from "lib/constants"; import { CustomSession } from "lib/types"; import { formatBytes, @@ -17,13 +17,17 @@ import { lookUp, } from "lib/utils"; import { PrismaService } from "modules/prisma/prisma.service"; +import { S3Service } from "modules/s3/s3.service"; import { join } from "path"; import { promisify } from "util"; @Injectable() export class RootService { private logger = new Logger(RootService.name); - constructor(private readonly prismaService: PrismaService) {} + constructor( + private readonly prismaService: PrismaService, + private readonly s3Service: S3Service + ) {} downloadFile(filename: string, res: Response) { if (!filename) { @@ -88,6 +92,7 @@ export class RootService { const isAudio = lookUp(file.filename).includes("audio"); const cannotDisplay = !isImage && !isVideo && !isAudio; const timezone = new Date().getTimezoneOffset() / 60; + let albumCover = null; if (process.env.UPLOADER === "s3") { baseUrl = @@ -95,21 +100,37 @@ export class RootService { "https://" + process.env.BUCKET_NAME + process.env.S3_ENDPOINT; oembed = `${baseUrl}/${slug}.json`; url = `${baseUrl}/${slug}.${ext}`; + await this.s3Service + .createInstance() + .getObject(process.env.S3_BUCKET_NAME, `${slug}.jpg`, (error) => { + if (error) { + albumCover = null; + } else { + albumCover = `${baseUrl}/${slug}.jpg`; + } + }); } else { baseUrl = `${protocol}://${req.headers.host}`; oembed = `${baseUrl}/${slug}.json`; url = `${baseUrl}/${slug}.${ext}`; + if (existsSync(join(uploadDir, `${slug}.jpg`))) { + albumCover = `${baseUrl}/${slug}.jpg`; + } else { + albumCover = null; + } } const { user: { embed_settings }, } = file; + const enabled = embed_settings?.enabled; + return { oembed, url, - title: embed_settings.enabled ? embed_settings?.title : null, - description: embed_settings.enabled ? embed_settings?.description : null, + title: enabled ? embed_settings?.title : null, + description: enabled ? embed_settings?.description : null, color: embed_settings?.color ?? generateRandomHexColor(), ogType: isVideo ? "video.other" : isImage ? "image" : "website", urlType: isVideo ? "video" : isAudio ? "audio" : "image", @@ -118,13 +139,14 @@ export class RootService { slug: file.slug + "." + file.filename.split(".").pop(), size: formatBytes(file.size), username: file.user.username, - embed_enabled: embed_settings?.enabled, + embed_enabled: enabled, views: vw, timestamp: formatDate(file.createdAt) + ` (UTC${timezone})`, isVideo, isImage, isAudio, cannotDisplay, + albumCover, }; } diff --git a/api/src/modules/s3/s3.service.ts b/api/src/modules/s3/s3.service.ts index 1b715a2..f7700ca 100644 --- a/api/src/modules/s3/s3.service.ts +++ b/api/src/modules/s3/s3.service.ts @@ -36,6 +36,10 @@ export class S3Service { }); } + createInstance() { + return this.s3; + } + createOEmbedJSON(oembed: Partial & { filename: string }) { const { author_name, author_url, provider_name, provider_url, filename } = oembed; diff --git a/api/src/modules/upload/upload.service.ts b/api/src/modules/upload/upload.service.ts index 22c01a1..255b1f2 100644 --- a/api/src/modules/upload/upload.service.ts +++ b/api/src/modules/upload/upload.service.ts @@ -14,6 +14,8 @@ import { generateRandomString, lookUp } from "lib/utils"; import { PrismaService } from "modules/prisma/prisma.service"; import { join } from "path"; import md5 from "md5"; +import { exec } from "node:child_process"; +import ffmpegPath from "ffmpeg-static"; @Injectable() export class UploadService { @@ -206,7 +208,7 @@ export class UploadService { } const name = decodeURIComponent(req.headers["x-file-name"] as string); - const size = req.headers["x-file-size"] as string; + let size = req.headers["x-file-size"] as string; const currentChunk = req.headers["x-current-chunk"] as string; const totalChunks = req.headers["x-total-chunks"] as string; @@ -253,17 +255,37 @@ export class UploadService { const { embed_settings } = user; - if ( - mimetype.includes("image") && - embed_settings && - embed_settings.enabled - ) { + if (mimetype.includes("image") && embed_settings?.enabled) { this.createOEmbedJSON({ filename: name, ...embed_settings, }); } + if (mimetype.includes("audio")) { + // get album cover if exists + const { stderr } = await exec( + `${ffmpegPath} -i ${join( + uploadDir, + `${slug}.${ext}` + )} -an -vcodec copy ${join(uploadDir, `${slug}.jpg`)}` + ); + + if (stderr) { + return; + } + + if ( + embed_settings?.enabled && + existsSync(join(uploadDir, `${slug}.jpg`)) + ) { + this.createOEmbedJSON({ + filename: name, + ...embed_settings, + }); + } + } + await this.prismaService.file.create({ data: { userId: user.id, diff --git a/api/src/modules/users/users.controller.ts b/api/src/modules/users/users.controller.ts index 77a26a8..94c44e6 100644 --- a/api/src/modules/users/users.controller.ts +++ b/api/src/modules/users/users.controller.ts @@ -32,7 +32,7 @@ export class UsersController { @SkipThrottle(false) @UseGuards(AuthGuard) @Post("verify/send") - async sendVerifyMail(@Request() req: ERequest) { + sendVerifyMail(@Request() req: ERequest) { return this.usersService.sendVerifyEmail( (req.session as CustomSession).userId ); @@ -40,32 +40,32 @@ export class UsersController { @UseGuards(AuthGuard) @Post("verify") - async verifyEmail(@Body() { token }: { token: string }) { + verifyEmail(@Body() { token }: { token: string }) { return this.usersService.verifyEmail(token); } @SkipThrottle(false) @Post("forgot-password") - async forgotPassword(@Body() { email }: { email: string }) { + forgotPassword(@Body() { email }: { email: string }) { return this.usersService.sendForgotPasswordEmail(email); } @SkipThrottle(false) @Post("check-token") - async checkToken(@Body() { token }: { token: string }) { + checkToken(@Body() { token }: { token: string }) { return this.usersService.checkToken(token); } @UseFilters(new HttpExceptionFilter()) @UsePipes(new ValidationPipe({ transform: true })) @Post("reset-password") - async resetPassword(@Body() { token, password }: ResetPasswordDTO) { + resetPassword(@Body() { token, password }: ResetPasswordDTO) { return this.usersService.resetPassword(token, password); } @UseGuards(AuthGuard) @Get("files") - async getFiles( + getFiles( @Request() req: ERequest, @Query("skip") skip: string, @Query("take") take: string, @@ -90,10 +90,7 @@ export class UsersController { @UseFilters(new HttpExceptionFilter()) @UsePipes(new ValidationPipe({ transform: true })) @Put("embed-settings") - async updateEmbedSettings( - @Request() req: ERequest, - @Body() body: EmbedSettingDTO - ) { + updateEmbedSettings(@Request() req: ERequest, @Body() body: EmbedSettingDTO) { return this.usersService.setEmbedSettings( body, (req.session as CustomSession).userId @@ -102,7 +99,7 @@ export class UsersController { @UseGuards(AuthGuard) @Get("embed-settings") - async getEmbedSettings(@Request() req: ERequest) { + getEmbedSettings(@Request() req: ERequest) { return this.usersService.getEmbedSettings( (req.session as CustomSession).userId ); @@ -113,7 +110,7 @@ export class UsersController { @UsePipes(new ValidationPipe({ transform: true })) @UseFilters(new HttpExceptionFilter()) @Put("change-password") - async changePassword( + changePassword( @Request() req: ERequest, @Body() { password, newPassword }: ChangePasswordDTO @@ -130,7 +127,7 @@ export class UsersController { @UsePipes(new ValidationPipe({ transform: true })) @UseFilters(new HttpExceptionFilter()) @Put("change-username") - async changeUsername( + changeUsername( @Body() { username, newUsername }: ChangeUsernameDTO ) { @@ -141,7 +138,7 @@ export class UsersController { @Throttle(1, 300) @UseGuards(AuthGuard) @Put("regenerate-api-key") - async regnerateApiKey(@Request() req: ERequest) { + regnerateApiKey(@Request() req: ERequest) { return this.usersService.regenerateApiKey( (req.session as CustomSession).userId ); @@ -149,9 +146,15 @@ export class UsersController { @UseGuards(AuthGuard) @Delete("delete-account") - async deleteAccount(@Request() req: ERequest) { + deleteAccount(@Request() req: ERequest) { return this.usersService.deleteAccount( (req.session as CustomSession).userId ); } + + @UseGuards(AuthGuard) + @Delete("wipe-files") + wipeFiles(@Request() req: ERequest) { + return this.usersService.wipeFiles((req.session as CustomSession).userId); + } } diff --git a/api/src/modules/users/users.service.ts b/api/src/modules/users/users.service.ts index e0ad59b..c24f32a 100644 --- a/api/src/modules/users/users.service.ts +++ b/api/src/modules/users/users.service.ts @@ -5,7 +5,7 @@ import { Logger, UnauthorizedException, } from "@nestjs/common"; -import { EmbedSettings, User } from "@prisma/client"; +import { EmbedSettings, File, User } from "@prisma/client"; import { PrismaClientKnownRequestError } from "@prisma/client/runtime"; import * as argon from "argon2"; import { MailService } from "../mail/mail.service"; @@ -29,9 +29,10 @@ import { FORGOT_PASSWORD_PREFIX, INVITE_PREFIX, rootDir, + uploadDir, } from "lib/constants"; import { join } from "path"; -import { readFile } from "fs/promises"; +import { readFile, unlink } from "fs/promises"; import { EmbedSettingDTO } from "./dto/EmbedSettingsDTO"; import cuid from "cuid"; @@ -49,12 +50,13 @@ export class UsersService implements IUserService { async findUser( username_or_email: string, - { byId, withPassword, totalUsed }: findUserOptions = { + { byId, withPassword, totalUsed, withFiles }: findUserOptions = { byId: false, withPassword: false, totalUsed: false, + withFiles: false, } - ): Promise { + ): Promise<(User & { files?: File[] }) | null> { if (!username_or_email) { throw new BadRequestException("Invalid request"); } @@ -68,6 +70,9 @@ export class UsersService implements IUserService { where: { id: username_or_email, }, + include: { + files: withFiles, + }, }); if (totalUsed) { const tmp = await this.prisma.file.aggregate({ @@ -87,6 +92,7 @@ export class UsersService implements IUserService { where: username_or_email.includes("@") ? { email: username_or_email } : { username: username_or_email }, + include: { files: withFiles }, }); if (totalUsed) { @@ -558,8 +564,31 @@ export class UsersService implements IUserService { return true; } + async wipeFiles(id: string) { + const files = await this.prisma.file.findMany({ where: { userId: id } }); + + const promises = files.map((file) => { + const ext = file.filename.split(".").pop(); + const filename = `${file.slug}.${ext}`; + if (file.mimetype.includes("audio")) { + return Promise.all([ + unlink(join(uploadDir, filename)), + unlink(join(uploadDir, `${file.slug}.jpg`)), + ]); + } + return unlink(join(uploadDir, filename)); + }); + + await Promise.all([ + this.prisma.file.deleteMany({ where: { userId: id } }), + ...promises, + ]).catch(() => {}); + + return true; + } + async deleteAccount(id: string) { - const user = await this.findUser(id, { byId: true }); + const user = await this.findUser(id, { byId: true, withFiles: true }); if (!user) { throw new UnauthorizedException("not authorized"); @@ -573,9 +602,11 @@ export class UsersService implements IUserService { throw new BadRequestException("You cannot delete root account"); } - await this.prisma.user.delete({ where: { id } }); + const wiped = await this.wipeFiles(id); - return true; + await this.prisma.user.delete({ where: { id } }).catch(() => {}); + + return wiped; } async sendForgotPasswordEmail(email: string) { @@ -590,7 +621,12 @@ export class UsersService implements IUserService { }); } - const isEmail = /\S+@\S+\.\S+/.test(email); + // RFC 5322 Official Standard + const emailRegex = new RegExp( + /(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])/g + ); + + const isEmail = emailRegex.test(email); if (!isEmail) { throw new BadRequestException({ diff --git a/api/views/index.hbs b/api/views/index.hbs index befb801..7fbb0f8 100644 --- a/api/views/index.hbs +++ b/api/views/index.hbs @@ -86,6 +86,14 @@ {{/if}} {{#if isAudio}} + {{#if albumCover}} + Album Cover + {{/if}}