refactor(server): upload config (#3148)

This commit is contained in:
Jason Rasmussen 2023-07-09 00:37:40 -04:00 committed by GitHub
parent 8349a28ed8
commit 398bd04ffd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 473 additions and 624 deletions

View File

@ -1,4 +1,4 @@
import { BadRequestException } from '@nestjs/common';
import { BadRequestException, UnauthorizedException } from '@nestjs/common';
import { AuthUserDto } from '../auth';
import { IAccessRepository } from './access.repository';
@ -25,6 +25,13 @@ export enum Permission {
export class AccessCore {
constructor(private repository: IAccessRepository) {}
requireUploadAccess(authUser: AuthUserDto | null): AuthUserDto {
if (!authUser || (authUser.isPublicUser && !authUser.isAllowUpload)) {
throw new UnauthorizedException();
}
return authUser;
}
async requirePermission(authUser: AuthUserDto, permission: Permission, ids: string[] | string) {
const hasAccess = await this.hasPermission(authUser, permission, ids);
if (!hasAccess) {

View File

@ -1,10 +1,10 @@
import { AssetEntity } from '@app/infra/entities';
import { BadRequestException, Inject } from '@nestjs/common';
import { DateTime } from 'luxon';
import { extname } from 'path';
import { AssetEntity } from '../../infra/entities/asset.entity';
import { AccessCore, IAccessRepository, Permission } from '../access';
import { AuthUserDto } from '../auth';
import { HumanReadableSize, usePagination } from '../domain.util';
import { AccessCore, IAccessRepository, Permission } from '../index';
import { ImmichReadStream, IStorageRepository } from '../storage';
import { IAssetRepository } from './asset.repository';
import { AssetIdsDto, DownloadArchiveInfo, DownloadDto, DownloadResponseDto, MemoryLaneDto } from './dto';
@ -12,6 +12,20 @@ import { MapMarkerDto } from './dto/map-marker.dto';
import { mapAsset, MapMarkerResponseDto } from './response-dto';
import { MemoryLaneResponseDto } from './response-dto/memory-lane-response.dto';
export enum UploadFieldName {
ASSET_DATA = 'assetData',
LIVE_PHOTO_DATA = 'livePhotoData',
SIDECAR_DATA = 'sidecarData',
PROFILE_DATA = 'file',
}
export interface UploadFile {
mimeType: string;
checksum: Buffer;
originalPath: string;
originalName: string;
}
export class AssetService {
private access: AccessCore;

View File

@ -2,6 +2,7 @@ export const ICryptoRepository = 'ICryptoRepository';
export interface ICryptoRepository {
randomBytes(size: number): Buffer;
randomUUID(): string;
hashFile(filePath: string): Promise<Buffer>;
hashSha256(data: string): string;
hashBcrypt(data: string | Buffer, saltOrRounds: string | number): Promise<string>;

View File

@ -1,21 +0,0 @@
import { validMimeTypes } from './domain.constant';
describe('valid mime types', () => {
it('should be a sorted list', () => {
expect(validMimeTypes).toEqual(validMimeTypes.sort());
});
it('should contain only unique values', () => {
expect(validMimeTypes).toEqual([...new Set(validMimeTypes)]);
});
it('should contain only image or video mime types', () => {
expect(validMimeTypes).toEqual(
validMimeTypes.filter((mimeType) => mimeType.startsWith('image/') || mimeType.startsWith('video/')),
);
});
it('should contain only lowercase mime types', () => {
expect(validMimeTypes).toEqual(validMimeTypes.map((mimeType) => mimeType.toLowerCase()));
});
});

View File

@ -28,7 +28,7 @@ export function assertMachineLearningEnabled() {
}
}
export const validMimeTypes = [
export const ASSET_MIME_TYPES = [
'image/3fr',
'image/ari',
'image/arw',
@ -106,11 +106,14 @@ export const validMimeTypes = [
'video/x-ms-wmv',
'video/x-msvideo',
];
export function isSupportedFileType(mimetype: string): boolean {
return validMimeTypes.includes(mimetype);
}
export function isSidecarFileType(mimeType: string): boolean {
return ['application/xml', 'text/xml'].includes(mimeType);
}
export const LIVE_PHOTO_MIME_TYPES = ASSET_MIME_TYPES;
export const SIDECAR_MIME_TYPES = ['application/xml', 'text/xml'];
export const PROFILE_MIME_TYPES = [
'image/jpeg',
'image/png',
'image/heic',
'image/heif',
'image/dng',
'image/webp',
'image/avif',
];

View File

@ -1,7 +1,7 @@
import { ApiProperty } from '@nestjs/swagger';
import { Express } from 'express';
import { UploadFieldName } from '../../asset/asset.service';
export class CreateProfileImageDto {
@ApiProperty({ type: 'string', format: 'binary' })
file!: Express.Multer.File;
[UploadFieldName.PROFILE_DATA]!: Express.Multer.File;
}

View File

@ -18,11 +18,10 @@ import {
UseInterceptors,
ValidationPipe,
} from '@nestjs/common';
import { FileFieldsInterceptor } from '@nestjs/platform-express';
import { ApiBody, ApiConsumes, ApiHeader, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { Response as Res } from 'express';
import { Authenticated, AuthUser, SharedLinkRoute } from '../../app.guard';
import { assetUploadOption, ImmichFile } from '../../config/asset-upload.config';
import { FileUploadInterceptor, ImmichFile, mapToUploadFile, Route } from '../../app.interceptor';
import { UUIDParamDto } from '../../controllers/dto/uuid-param.dto';
import FileNotEmptyValidator from '../validation/file-not-empty-validator';
import { AssetService } from './asset.service';
@ -30,7 +29,7 @@ import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
import { AssetSearchDto } from './dto/asset-search.dto';
import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CreateAssetDto, ImportAssetDto, mapToUploadFile } from './dto/create-asset.dto';
import { CreateAssetDto, ImportAssetDto } from './dto/create-asset.dto';
import { DeleteAssetDto } from './dto/delete-asset.dto';
import { DeviceIdDto } from './dto/device-id.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
@ -56,23 +55,14 @@ interface UploadFiles {
}
@ApiTags('Asset')
@Controller('asset')
@Controller(Route.ASSET)
@Authenticated()
export class AssetController {
constructor(private assetService: AssetService) {}
@SharedLinkRoute()
@Post('upload')
@UseInterceptors(
FileFieldsInterceptor(
[
{ name: 'assetData', maxCount: 1 },
{ name: 'livePhotoData', maxCount: 1 },
{ name: 'sidecarData', maxCount: 1 },
],
assetUploadOption,
),
)
@UseInterceptors(FileUploadInterceptor)
@ApiConsumes('multipart/form-data')
@ApiBody({
description: 'Asset Upload Information',

View File

@ -1,8 +1,8 @@
import { AuthUserDto, IJobRepository, JobName } from '@app/domain';
import { AuthUserDto, IJobRepository, JobName, UploadFile } from '@app/domain';
import { AssetEntity, UserEntity } from '@app/infra/entities';
import { parse } from 'node:path';
import { IAssetRepository } from './asset-repository';
import { CreateAssetDto, ImportAssetDto, UploadFile } from './dto/create-asset.dto';
import { CreateAssetDto, ImportAssetDto } from './dto/create-asset.dto';
export class AssetCore {
constructor(private repository: IAssetRepository, private jobRepository: IJobRepository) {}

View File

@ -1,13 +0,0 @@
import { AssetEntity, ExifEntity } from '@app/infra/entities';
import { Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AssetRepository, IAssetRepository } from './asset-repository';
import { AssetController } from './asset.controller';
import { AssetService } from './asset.service';
@Module({
imports: [TypeOrmModule.forFeature([AssetEntity, ExifEntity])],
controllers: [AssetController],
providers: [AssetService, { provide: IAssetRepository, useClass: AssetRepository }],
})
export class AssetModule {}

View File

@ -1,6 +1,16 @@
import { ICryptoRepository, IJobRepository, IStorageRepository, JobName } from '@app/domain';
import {
ASSET_MIME_TYPES,
ICryptoRepository,
IJobRepository,
IStorageRepository,
JobName,
LIVE_PHOTO_MIME_TYPES,
PROFILE_MIME_TYPES,
SIDECAR_MIME_TYPES,
UploadFieldName,
} from '@app/domain';
import { AssetEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import { BadRequestException, UnauthorizedException } from '@nestjs/common';
import {
assetEntityStub,
authStub,
@ -117,6 +127,43 @@ const _getArchivedAssetsCountByUserId = (): AssetCountByUserIdResponseDto => {
return result;
};
const uploadFile = {
nullAuth: {
authUser: null,
fieldName: UploadFieldName.ASSET_DATA,
file: {
mimeType: 'image/jpeg',
checksum: Buffer.from('checksum', 'utf8'),
originalPath: 'upload/admin/image.jpeg',
originalName: 'image.jpeg',
},
},
mimeType: (fieldName: UploadFieldName, mimeType: string) => {
return {
authUser: authStub.admin,
fieldName,
file: {
mimeType,
checksum: Buffer.from('checksum', 'utf8'),
originalPath: 'upload/admin/image.jpeg',
originalName: 'image.jpeg',
},
};
},
filename: (fieldName: UploadFieldName, filename: string) => {
return {
authUser: authStub.admin,
fieldName,
file: {
mimeType: 'image/jpeg',
checksum: Buffer.from('checksum', 'utf8'),
originalPath: `upload/admin/${filename}`,
originalName: filename,
},
};
},
};
describe('AssetService', () => {
let sut: AssetService;
let a: Repository<AssetEntity>; // TO BE DELETED AFTER FINISHED REFACTORING
@ -165,6 +212,112 @@ describe('AssetService', () => {
.mockResolvedValue(assetEntityStub.livePhotoMotionAsset);
});
const tests = [
{ label: 'asset', fieldName: UploadFieldName.ASSET_DATA, mimeTypes: ASSET_MIME_TYPES },
{ label: 'live photo', fieldName: UploadFieldName.LIVE_PHOTO_DATA, mimeTypes: LIVE_PHOTO_MIME_TYPES },
{ label: 'sidecar', fieldName: UploadFieldName.SIDECAR_DATA, mimeTypes: SIDECAR_MIME_TYPES },
{ label: 'profile', fieldName: UploadFieldName.PROFILE_DATA, mimeTypes: PROFILE_MIME_TYPES },
];
for (const { label, fieldName, mimeTypes } of tests) {
describe(`${label} mime types linting`, () => {
it('should be a sorted list', () => {
expect(mimeTypes).toEqual(mimeTypes.sort());
});
it('should contain only unique values', () => {
expect(mimeTypes).toEqual([...new Set(mimeTypes)]);
});
if (fieldName !== UploadFieldName.SIDECAR_DATA) {
it('should contain only image or video mime types', () => {
expect(mimeTypes).toEqual(
mimeTypes.filter((mimeType) => mimeType.startsWith('image/') || mimeType.startsWith('video/')),
);
});
}
it('should contain only lowercase mime types', () => {
expect(mimeTypes).toEqual(mimeTypes.map((mimeType) => mimeType.toLowerCase()));
});
});
}
describe('canUpload', () => {
it('should require an authenticated user', () => {
expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException);
});
it('should accept all accepted mime types', () => {
for (const { fieldName, mimeTypes } of tests) {
for (const mimeType of mimeTypes) {
expect(sut.canUploadFile(uploadFile.mimeType(fieldName, mimeType))).toEqual(true);
}
}
});
it('should reject other mime types', () => {
for (const { fieldName, mimeType } of [
{ fieldName: UploadFieldName.ASSET_DATA, mimeType: 'application/html' },
{ fieldName: UploadFieldName.LIVE_PHOTO_DATA, mimeType: 'application/html' },
{ fieldName: UploadFieldName.PROFILE_DATA, mimeType: 'application/html' },
{ fieldName: UploadFieldName.SIDECAR_DATA, mimeType: 'image/jpeg' },
]) {
expect(() => sut.canUploadFile(uploadFile.mimeType(fieldName, mimeType))).toThrowError(BadRequestException);
}
});
});
describe('getUploadFilename', () => {
it('should require authentication', () => {
expect(() => sut.getUploadFilename(uploadFile.nullAuth)).toThrowError(UnauthorizedException);
});
it('should be the original extension for asset upload', () => {
expect(sut.getUploadFilename(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
'random-uuid.jpg',
);
});
it('should be the mov extension for live photo upload', () => {
expect(sut.getUploadFilename(uploadFile.filename(UploadFieldName.LIVE_PHOTO_DATA, 'image.mp4'))).toEqual(
'random-uuid.mov',
);
});
it('should be the xmp extension for sidecar upload', () => {
expect(sut.getUploadFilename(uploadFile.filename(UploadFieldName.SIDECAR_DATA, 'image.html'))).toEqual(
'random-uuid.xmp',
);
});
it('should be the original extension for profile upload', () => {
expect(sut.getUploadFilename(uploadFile.filename(UploadFieldName.PROFILE_DATA, 'image.jpg'))).toEqual(
'random-uuid.jpg',
);
});
});
describe('getUploadFolder', () => {
it('should require authentication', () => {
expect(() => sut.getUploadFolder(uploadFile.nullAuth)).toThrowError(UnauthorizedException);
});
it('should return profile for profile uploads', () => {
expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.PROFILE_DATA, 'image.jpg'))).toEqual(
'upload/profile/admin_id',
);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/profile/admin_id');
});
it('should return upload for everything else', () => {
expect(sut.getUploadFolder(uploadFile.filename(UploadFieldName.ASSET_DATA, 'image.jpg'))).toEqual(
'upload/upload/admin_id',
);
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/upload/admin_id');
});
});
describe('uploadFile', () => {
it('should handle a file upload', async () => {
const assetEntity = _getAsset_1();

View File

@ -1,17 +1,24 @@
import {
AccessCore,
AssetResponseDto,
ASSET_MIME_TYPES,
AuthUserDto,
getLivePhotoMotionFilename,
IAccessRepository,
ICryptoRepository,
IJobRepository,
isSupportedFileType,
IStorageRepository,
JobName,
LIVE_PHOTO_MIME_TYPES,
mapAsset,
mapAssetWithoutExif,
Permission,
PROFILE_MIME_TYPES,
SIDECAR_MIME_TYPES,
StorageCore,
StorageFolder,
UploadFieldName,
UploadFile,
} from '@app/domain';
import { AssetEntity, AssetType } from '@app/infra/entities';
import {
@ -27,16 +34,18 @@ import { Response as Res } from 'express';
import { constants, createReadStream } from 'fs';
import fs from 'fs/promises';
import mime from 'mime-types';
import path from 'path';
import path, { extname } from 'path';
import sanitize from 'sanitize-filename';
import { pipeline } from 'stream/promises';
import { QueryFailedError, Repository } from 'typeorm';
import { UploadRequest } from '../../app.interceptor';
import { IAssetRepository } from './asset-repository';
import { AssetCore } from './asset.core';
import { AssetBulkUploadCheckDto } from './dto/asset-check.dto';
import { AssetSearchDto } from './dto/asset-search.dto';
import { CheckDuplicateAssetDto } from './dto/check-duplicate-asset.dto';
import { CheckExistingAssetsDto } from './dto/check-existing-assets.dto';
import { CreateAssetDto, ImportAssetDto, UploadFile } from './dto/create-asset.dto';
import { CreateAssetDto, ImportAssetDto } from './dto/create-asset.dto';
import { DeleteAssetDto } from './dto/delete-asset.dto';
import { GetAssetByTimeBucketDto } from './dto/get-asset-by-time-bucket.dto';
import { GetAssetCountByTimeBucketDto } from './dto/get-asset-count-by-time-bucket.dto';
@ -72,6 +81,7 @@ export class AssetService {
readonly logger = new Logger(AssetService.name);
private assetCore: AssetCore;
private access: AccessCore;
private storageCore = new StorageCore();
constructor(
@Inject(IAccessRepository) accessRepository: IAccessRepository,
@ -85,6 +95,68 @@ export class AssetService {
this.access = new AccessCore(accessRepository);
}
canUploadFile({ authUser, fieldName, file }: UploadRequest): true {
this.access.requireUploadAccess(authUser);
switch (fieldName) {
case UploadFieldName.ASSET_DATA:
if (ASSET_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.LIVE_PHOTO_DATA:
if (LIVE_PHOTO_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.SIDECAR_DATA:
if (SIDECAR_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
case UploadFieldName.PROFILE_DATA:
if (PROFILE_MIME_TYPES.includes(file.mimeType)) {
return true;
}
break;
}
const ext = extname(file.originalName);
this.logger.error(`Unsupported file type ${ext} file MIME type ${file.mimeType}`);
throw new BadRequestException(`Unsupported file type ${ext}`);
}
getUploadFilename({ authUser, fieldName, file }: UploadRequest): string {
this.access.requireUploadAccess(authUser);
const originalExt = extname(file.originalName);
const lookup = {
[UploadFieldName.ASSET_DATA]: originalExt,
[UploadFieldName.LIVE_PHOTO_DATA]: '.mov',
[UploadFieldName.SIDECAR_DATA]: '.xmp',
[UploadFieldName.PROFILE_DATA]: originalExt,
};
return sanitize(`${this.cryptoRepository.randomUUID()}${lookup[fieldName]}`);
}
getUploadFolder({ authUser, fieldName }: UploadRequest): string {
authUser = this.access.requireUploadAccess(authUser);
let folder = this.storageCore.getFolderLocation(StorageFolder.UPLOAD, authUser.id);
if (fieldName === UploadFieldName.PROFILE_DATA) {
folder = this.storageCore.getFolderLocation(StorageFolder.PROFILE, authUser.id);
}
this.storageRepository.mkdirSync(folder);
return folder;
}
public async uploadFile(
authUser: AuthUserDto,
dto: CreateAssetDto,
@ -136,9 +208,9 @@ export class AssetService {
sidecarPath: dto.sidecarPath ? path.resolve(dto.sidecarPath) : undefined,
};
const assetPathType = mime.lookup(dto.assetPath) as string;
if (!isSupportedFileType(assetPathType)) {
throw new BadRequestException(`Unsupported file type ${assetPathType}`);
const mimeType = mime.lookup(dto.assetPath) as string;
if (!ASSET_MIME_TYPES.includes(mimeType)) {
throw new BadRequestException(`Unsupported file type ${mimeType}`);
}
if (dto.sidecarPath) {
@ -164,7 +236,7 @@ export class AssetService {
const assetFile: UploadFile = {
checksum: await this.cryptoRepository.hashFile(dto.assetPath),
mimeType: assetPathType,
mimeType,
originalPath: dto.assetPath,
originalName: path.parse(dto.assetPath).name,
};

View File

@ -1,9 +1,8 @@
import { toBoolean, toSanitized } from '@app/domain';
import { toBoolean, toSanitized, UploadFieldName } from '@app/domain';
import { AssetType } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger';
import { Transform } from 'class-transformer';
import { IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator';
import { ImmichFile } from '../../../config/asset-upload.config';
export class CreateAssetBase {
@IsNotEmpty()
@ -50,13 +49,13 @@ export class CreateAssetDto extends CreateAssetBase {
// The properties below are added to correctly generate the API docs
// and client SDKs. Validation should be handled in the controller.
@ApiProperty({ type: 'string', format: 'binary' })
assetData!: any;
[UploadFieldName.ASSET_DATA]!: any;
@ApiProperty({ type: 'string', format: 'binary' })
livePhotoData?: any;
@ApiProperty({ type: 'string', format: 'binary', required: false })
[UploadFieldName.LIVE_PHOTO_DATA]?: any;
@ApiProperty({ type: 'string', format: 'binary' })
sidecarData?: any;
@ApiProperty({ type: 'string', format: 'binary', required: false })
[UploadFieldName.SIDECAR_DATA]?: any;
}
export class ImportAssetDto extends CreateAssetBase {
@ -75,19 +74,3 @@ export class ImportAssetDto extends CreateAssetBase {
@Transform(toSanitized)
sidecarPath?: string;
}
export interface UploadFile {
mimeType: string;
checksum: Buffer;
originalPath: string;
originalName: string;
}
export function mapToUploadFile(file: ImmichFile): UploadFile {
return {
checksum: file.checksum,
mimeType: file.mimetype,
originalPath: file.path,
originalName: file.originalname,
};
}

View File

@ -2,9 +2,7 @@ import { FileValidator, Injectable } from '@nestjs/common';
@Injectable()
export default class FileNotEmptyValidator extends FileValidator {
requiredFields: string[];
constructor(requiredFields: string[]) {
constructor(private requiredFields: string[]) {
super({});
this.requiredFields = requiredFields;
}
@ -14,9 +12,7 @@ export default class FileNotEmptyValidator extends FileValidator {
return false;
}
return this.requiredFields.every((field) => {
return files[field];
});
return this.requiredFields.every((field) => files[field]);
}
buildErrorMessage(): string {

View File

@ -0,0 +1,168 @@
import { AuthUserDto, UploadFieldName, UploadFile } from '@app/domain';
import { CallHandler, ExecutionContext, Injectable, Logger, NestInterceptor } from '@nestjs/common';
import { PATH_METADATA } from '@nestjs/common/constants';
import { Reflector } from '@nestjs/core';
import { transformException } from '@nestjs/platform-express/multer/multer/multer.utils';
import { createHash } from 'crypto';
import { NextFunction, RequestHandler } from 'express';
import multer, { diskStorage, StorageEngine } from 'multer';
import { Observable } from 'rxjs';
import { AssetService } from './api-v1/asset/asset.service';
import { AuthRequest } from './app.guard';
export enum Route {
ASSET = 'asset',
USER = 'user',
}
export interface ImmichFile extends Express.Multer.File {
/** sha1 hash of file */
checksum: Buffer;
}
export function mapToUploadFile(file: ImmichFile): UploadFile {
return {
checksum: file.checksum,
mimeType: file.mimetype,
originalPath: file.path,
originalName: Buffer.from(file.originalname, 'latin1').toString('utf8'),
};
}
type DiskStorageCallback = (error: Error | null, result: string) => void;
interface Callback<T> {
(error: Error): void;
(error: null, result: T): void;
}
const callbackify = async <T>(fn: (...args: any[]) => T, callback: Callback<T>) => {
try {
return callback(null, await fn());
} catch (error: Error | any) {
return callback(error);
}
};
export interface UploadRequest {
authUser: AuthUserDto | null;
fieldName: UploadFieldName;
file: UploadFile;
}
const asRequest = (req: AuthRequest, file: Express.Multer.File) => {
return {
authUser: req.user || null,
fieldName: file.fieldname as UploadFieldName,
file: mapToUploadFile(file as ImmichFile),
};
};
@Injectable()
export class FileUploadInterceptor implements NestInterceptor {
private logger = new Logger(FileUploadInterceptor.name);
private handlers: {
userProfile: RequestHandler;
assetUpload: RequestHandler;
};
private defaultStorage: StorageEngine;
constructor(private reflect: Reflector, private assetService: AssetService) {
this.defaultStorage = diskStorage({
filename: this.filename.bind(this),
destination: this.destination.bind(this),
});
const instance = multer({
fileFilter: this.fileFilter.bind(this),
storage: {
_handleFile: this.handleFile.bind(this),
_removeFile: this.removeFile.bind(this),
},
});
this.handlers = {
userProfile: instance.single(UploadFieldName.PROFILE_DATA),
assetUpload: instance.fields([
{ name: UploadFieldName.ASSET_DATA, maxCount: 1 },
{ name: UploadFieldName.LIVE_PHOTO_DATA, maxCount: 1 },
{ name: UploadFieldName.SIDECAR_DATA, maxCount: 1 },
]),
};
}
async intercept(context: ExecutionContext, next: CallHandler<any>): Promise<Observable<any>> {
const ctx = context.switchToHttp();
const route = this.reflect.get<string>(PATH_METADATA, context.getClass());
const handler: RequestHandler | null = this.getHandler(route as Route);
if (handler) {
await new Promise<void>((resolve, reject) => {
const next: NextFunction = (error) => (error ? reject(transformException(error)) : resolve());
handler(ctx.getRequest(), ctx.getResponse(), next);
});
} else {
this.logger.warn(`Skipping invalid file upload route: ${route}`);
}
return next.handle();
}
private fileFilter(req: AuthRequest, file: Express.Multer.File, callback: multer.FileFilterCallback) {
return callbackify(() => this.assetService.canUploadFile(asRequest(req, file)), callback);
}
private filename(req: AuthRequest, file: Express.Multer.File, callback: DiskStorageCallback) {
return callbackify(() => this.assetService.getUploadFilename(asRequest(req, file)), callback as Callback<string>);
}
private destination(req: AuthRequest, file: Express.Multer.File, callback: DiskStorageCallback) {
return callbackify(() => this.assetService.getUploadFolder(asRequest(req, file)), callback as Callback<string>);
}
private handleFile(req: AuthRequest, file: Express.Multer.File, callback: Callback<Partial<ImmichFile>>) {
if (!this.isAssetUploadFile(file)) {
this.defaultStorage._handleFile(req, file, callback);
return;
}
const hash = createHash('sha1');
file.stream.on('data', (chunk) => hash.update(chunk));
this.defaultStorage._handleFile(req, file, (error, info) => {
if (error) {
hash.destroy();
callback(error);
} else {
callback(null, { ...info, checksum: hash.digest() });
}
});
}
private removeFile(req: AuthRequest, file: Express.Multer.File, callback: (error: Error | null) => void) {
this.defaultStorage._removeFile(req, file, callback);
}
private isAssetUploadFile(file: Express.Multer.File) {
switch (file.fieldname as UploadFieldName) {
case UploadFieldName.ASSET_DATA:
case UploadFieldName.LIVE_PHOTO_DATA:
return true;
}
return false;
}
private getHandler(route: Route) {
switch (route) {
case Route.ASSET:
return this.handlers.assetUpload;
case Route.USER:
return this.handlers.userProfile;
default:
return null;
}
}
}

View File

@ -1,11 +1,16 @@
import { DomainModule } from '@app/domain';
import { InfraModule } from '@app/infra';
import { AssetEntity, ExifEntity } from '@app/infra/entities';
import { Module } from '@nestjs/common';
import { APP_GUARD } from '@nestjs/core';
import { ScheduleModule } from '@nestjs/schedule';
import { TypeOrmModule } from '@nestjs/typeorm';
import { AlbumModule } from './api-v1/album/album.module';
import { AssetModule } from './api-v1/asset/asset.module';
import { AssetRepository, IAssetRepository } from './api-v1/asset/asset-repository';
import { AssetController as AssetControllerV1 } from './api-v1/asset/asset.controller';
import { AssetService } from './api-v1/asset/asset.service';
import { AppGuard } from './app.guard';
import { FileUploadInterceptor } from './app.interceptor';
import { AppService } from './app.service';
import {
AlbumController,
@ -29,11 +34,12 @@ import {
imports: [
//
DomainModule.register({ imports: [InfraModule] }),
AssetModule,
AlbumModule,
ScheduleModule.forRoot(),
TypeOrmModule.forFeature([AssetEntity, ExifEntity]),
],
controllers: [
AssetControllerV1,
AppController,
AlbumController,
APIKeyController,
@ -53,8 +59,11 @@ import {
providers: [
//
{ provide: APP_GUARD, useExisting: AppGuard },
{ provide: IAssetRepository, useClass: AssetRepository },
AppGuard,
AppService,
AssetService,
FileUploadInterceptor,
],
})
export class AppModule {}

View File

@ -34,10 +34,6 @@ export const asStreamableFile = ({ stream, type, length }: ImmichReadStream) =>
return new StreamableFile(stream, { type, length });
};
export function patchFormData(latin1: string) {
return Buffer.from(latin1, 'latin1').toString('utf8');
}
function sortKeys<T extends object>(obj: T): T {
if (!obj) {
return obj;

View File

@ -1,222 +0,0 @@
import { Request } from 'express';
import * as fs from 'fs';
import { AuthRequest } from '../app.guard';
import { multerUtils } from './asset-upload.config';
const { fileFilter, destination, filename } = multerUtils;
const mock = {
req: {} as Request,
userRequest: {
user: {
id: 'test-user',
},
body: {
deviceId: 'test-device',
fileExtension: '.jpg',
},
} as AuthRequest,
file: { originalname: 'test.jpg' } as Express.Multer.File,
};
jest.mock('fs');
describe('assetUploadOption', () => {
let callback: jest.Mock;
let existsSync: jest.Mock;
let mkdirSync: jest.Mock;
beforeEach(() => {
jest.mock('fs');
mkdirSync = fs.mkdirSync as jest.Mock;
existsSync = fs.existsSync as jest.Mock;
callback = jest.fn();
existsSync.mockImplementation(() => true);
});
afterEach(() => {
jest.resetModules();
});
describe('fileFilter', () => {
it('should require a user', () => {
fileFilter(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
for (const { mimetype, extension } of [
// Please ensure this list is sorted.
{ mimetype: 'image/3fr', extension: '3fr' },
{ mimetype: 'image/ari', extension: 'ari' },
{ mimetype: 'image/arw', extension: 'arw' },
{ mimetype: 'image/avif', extension: 'avif' },
{ mimetype: 'image/cap', extension: 'cap' },
{ mimetype: 'image/cin', extension: 'cin' },
{ mimetype: 'image/cr2', extension: 'cr2' },
{ mimetype: 'image/cr3', extension: 'cr3' },
{ mimetype: 'image/crw', extension: 'crw' },
{ mimetype: 'image/dcr', extension: 'dcr' },
{ mimetype: 'image/dng', extension: 'dng' },
{ mimetype: 'image/erf', extension: 'erf' },
{ mimetype: 'image/fff', extension: 'fff' },
{ mimetype: 'image/gif', extension: 'gif' },
{ mimetype: 'image/heic', extension: 'heic' },
{ mimetype: 'image/heif', extension: 'heif' },
{ mimetype: 'image/iiq', extension: 'iiq' },
{ mimetype: 'image/jpeg', extension: 'jpeg' },
{ mimetype: 'image/jpeg', extension: 'jpg' },
{ mimetype: 'image/jxl', extension: 'jxl' },
{ mimetype: 'image/k25', extension: 'k25' },
{ mimetype: 'image/kdc', extension: 'kdc' },
{ mimetype: 'image/mrw', extension: 'mrw' },
{ mimetype: 'image/nef', extension: 'nef' },
{ mimetype: 'image/orf', extension: 'orf' },
{ mimetype: 'image/ori', extension: 'ori' },
{ mimetype: 'image/pef', extension: 'pef' },
{ mimetype: 'image/png', extension: 'png' },
{ mimetype: 'image/raf', extension: 'raf' },
{ mimetype: 'image/raw', extension: 'raw' },
{ mimetype: 'image/rwl', extension: 'rwl' },
{ mimetype: 'image/sr2', extension: 'sr2' },
{ mimetype: 'image/srf', extension: 'srf' },
{ mimetype: 'image/srw', extension: 'srw' },
{ mimetype: 'image/tiff', extension: 'tiff' },
{ mimetype: 'image/webp', extension: 'webp' },
{ mimetype: 'image/x-adobe-dng', extension: 'dng' },
{ mimetype: 'image/x-arriflex-ari', extension: 'ari' },
{ mimetype: 'image/x-canon-cr2', extension: 'cr2' },
{ mimetype: 'image/x-canon-cr3', extension: 'cr3' },
{ mimetype: 'image/x-canon-crw', extension: 'crw' },
{ mimetype: 'image/x-epson-erf', extension: 'erf' },
{ mimetype: 'image/x-fuji-raf', extension: 'raf' },
{ mimetype: 'image/x-hasselblad-3fr', extension: '3fr' },
{ mimetype: 'image/x-hasselblad-fff', extension: 'fff' },
{ mimetype: 'image/x-kodak-dcr', extension: 'dcr' },
{ mimetype: 'image/x-kodak-k25', extension: 'k25' },
{ mimetype: 'image/x-kodak-kdc', extension: 'kdc' },
{ mimetype: 'image/x-leica-rwl', extension: 'rwl' },
{ mimetype: 'image/x-minolta-mrw', extension: 'mrw' },
{ mimetype: 'image/x-nikon-nef', extension: 'nef' },
{ mimetype: 'image/x-olympus-orf', extension: 'orf' },
{ mimetype: 'image/x-olympus-ori', extension: 'ori' },
{ mimetype: 'image/x-panasonic-raw', extension: 'raw' },
{ mimetype: 'image/x-pentax-pef', extension: 'pef' },
{ mimetype: 'image/x-phantom-cin', extension: 'cin' },
{ mimetype: 'image/x-phaseone-cap', extension: 'cap' },
{ mimetype: 'image/x-phaseone-iiq', extension: 'iiq' },
{ mimetype: 'image/x-samsung-srw', extension: 'srw' },
{ mimetype: 'image/x-sigma-x3f', extension: 'x3f' },
{ mimetype: 'image/x-sony-arw', extension: 'arw' },
{ mimetype: 'image/x-sony-sr2', extension: 'sr2' },
{ mimetype: 'image/x-sony-srf', extension: 'srf' },
{ mimetype: 'image/x3f', extension: 'x3f' },
{ mimetype: 'video/3gpp', extension: '3gp' },
{ mimetype: 'video/avi', extension: 'avi' },
{ mimetype: 'video/mp2t', extension: 'm2ts' },
{ mimetype: 'video/mp2t', extension: 'mts' },
{ mimetype: 'video/mp4', extension: 'mp4' },
{ mimetype: 'video/mpeg', extension: 'mpg' },
{ mimetype: 'video/msvideo', extension: 'avi' },
{ mimetype: 'video/quicktime', extension: 'mov' },
{ mimetype: 'video/vnd.avi', extension: 'avi' },
{ mimetype: 'video/webm', extension: 'webm' },
{ mimetype: 'video/x-flv', extension: 'flv' },
{ mimetype: 'video/x-matroska', extension: 'mkv' },
{ mimetype: 'video/x-ms-wmv', extension: 'wmv' },
{ mimetype: 'video/x-msvideo', extension: 'avi' },
]) {
const name = `test.${extension}`;
it(`should allow ${name} (${mimetype})`, async () => {
fileFilter(mock.userRequest, { mimetype, originalname: name }, callback);
expect(callback).toHaveBeenCalledWith(null, true);
});
}
it('should not allow unknown types', async () => {
const file = { mimetype: 'application/html', originalname: 'test.html' } as any;
const callback = jest.fn();
fileFilter(mock.userRequest, file, callback);
expect(callback).toHaveBeenCalled();
const [error, accepted] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(accepted).toBe(false);
});
});
describe('destination', () => {
it('should require a user', () => {
destination(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
it('should create non-existing directories', () => {
existsSync.mockImplementation(() => false);
destination(mock.userRequest, mock.file, callback);
expect(existsSync).toHaveBeenCalled();
expect(mkdirSync).toHaveBeenCalled();
});
it('should return the destination', () => {
destination(mock.userRequest, mock.file, callback);
expect(mkdirSync).not.toHaveBeenCalled();
expect(callback).toHaveBeenCalledWith(null, 'upload/upload/test-user');
});
});
describe('filename', () => {
it('should require a user', () => {
filename(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
it('should return the filename', () => {
filename(mock.userRequest, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeNull();
expect(name.endsWith('.jpg')).toBeTruthy();
});
it('should sanitize the filename', () => {
const body = { ...mock.userRequest.body, fileExtension: '.jp\u0000g' };
const request = { ...mock.userRequest, body } as Request;
filename(request, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeNull();
expect(name.endsWith(mock.userRequest.body.fileExtension)).toBeTruthy();
});
it('should not change the casing of the extension', () => {
// Case is deliberately mixed to cover both .upper() and .lower()
const body = { ...mock.userRequest.body, fileExtension: '.JpEg' };
const request = { ...mock.userRequest, body } as Request;
filename(request, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeNull();
expect(name.endsWith(body.fileExtension)).toBeTruthy();
});
});
});

View File

@ -1,109 +0,0 @@
import { AuthUserDto, isSidecarFileType, isSupportedFileType } from '@app/domain';
import { StorageCore, StorageFolder } from '@app/domain/storage';
import { BadRequestException, Logger, UnauthorizedException } from '@nestjs/common';
import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface';
import { createHash, randomUUID } from 'crypto';
import { existsSync, mkdirSync } from 'fs';
import { diskStorage, StorageEngine } from 'multer';
import { extname } from 'path';
import sanitize from 'sanitize-filename';
import { AuthRequest } from '../app.guard';
import { patchFormData } from '../app.utils';
export interface ImmichFile extends Express.Multer.File {
/** sha1 hash of file */
checksum: Buffer;
}
export const assetUploadOption: MulterOptions = {
fileFilter,
storage: customStorage(),
};
const storageCore = new StorageCore();
export function customStorage(): StorageEngine {
const storage = diskStorage({ destination, filename });
return {
_handleFile(req, file, callback) {
const hash = createHash('sha1');
file.stream.on('data', (chunk) => hash.update(chunk));
storage._handleFile(req, file, (error, response) => {
if (error) {
hash.destroy();
callback(error);
} else {
callback(null, { ...response, checksum: hash.digest() } as ImmichFile);
}
});
},
_removeFile(req, file, callback) {
storage._removeFile(req, file, callback);
},
};
}
export const multerUtils = { fileFilter, filename, destination };
const logger = new Logger('AssetUploadConfig');
function fileFilter(req: AuthRequest, file: any, cb: any) {
if (!req.user || (req.user.isPublicUser && !req.user.isAllowUpload)) {
return cb(new UnauthorizedException());
}
if (isSupportedFileType(file.mimetype)) {
cb(null, true);
return;
}
// Additionally support XML but only for sidecar files.
if (file.fieldname === 'sidecarData' && isSidecarFileType(file.mimetype)) {
return cb(null, true);
}
logger.error(`Unsupported file type ${extname(file.originalname)} file MIME type ${file.mimetype}`);
cb(new BadRequestException(`Unsupported file type ${extname(file.originalname)}`), false);
}
function destination(req: AuthRequest, file: Express.Multer.File, cb: any) {
if (!req.user || (req.user.isPublicUser && !req.user.isAllowUpload)) {
return cb(new UnauthorizedException());
}
const user = req.user as AuthUserDto;
const uploadFolder = storageCore.getFolderLocation(StorageFolder.UPLOAD, user.id);
if (!existsSync(uploadFolder)) {
mkdirSync(uploadFolder, { recursive: true });
}
// Save original to disk
cb(null, uploadFolder);
}
function filename(req: AuthRequest, file: Express.Multer.File, cb: any) {
if (!req.user || (req.user.isPublicUser && !req.user.isAllowUpload)) {
return cb(new UnauthorizedException());
}
file.originalname = patchFormData(file.originalname);
const fileNameUUID = randomUUID();
if (file.fieldname === 'livePhotoData') {
const livePhotoFileName = `${fileNameUUID}.mov`;
return cb(null, sanitize(livePhotoFileName));
}
if (file.fieldname === 'sidecarData') {
const sidecarFileName = `${fileNameUUID}.xmp`;
return cb(null, sanitize(sidecarFileName));
}
const fileName = `${fileNameUUID}${req.body['fileExtension']}`;
return cb(null, sanitize(fileName));
}

View File

@ -1,115 +0,0 @@
import { Request } from 'express';
import * as fs from 'fs';
import { AuthRequest } from '../app.guard';
import { multerUtils } from './profile-image-upload.config';
const { fileFilter, destination, filename } = multerUtils;
const mock = {
req: {} as Request,
userRequest: {
user: {
id: 'test-user',
},
} as AuthRequest,
file: { originalname: 'test.jpg' } as Express.Multer.File,
};
jest.mock('fs');
describe('profileImageUploadOption', () => {
let callback: jest.Mock;
let existsSync: jest.Mock;
let mkdirSync: jest.Mock;
beforeEach(() => {
jest.mock('fs');
mkdirSync = fs.mkdirSync as jest.Mock;
existsSync = fs.existsSync as jest.Mock;
callback = jest.fn();
existsSync.mockImplementation(() => true);
});
afterEach(() => {
jest.resetModules();
});
describe('fileFilter', () => {
it('should require a user', () => {
fileFilter(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
it('should allow images', async () => {
const file = { mimetype: 'image/jpeg', originalname: 'test.jpg' } as any;
fileFilter(mock.userRequest, file, callback);
expect(callback).toHaveBeenCalledWith(null, true);
});
it('should not allow gifs', async () => {
const file = { mimetype: 'image/gif', originalname: 'test.gif' } as any;
const callback = jest.fn();
fileFilter(mock.userRequest, file, callback);
expect(callback).toHaveBeenCalled();
const [error, accepted] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(accepted).toBe(false);
});
});
describe('destination', () => {
it('should require a user', () => {
destination(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
it('should create non-existing directories', () => {
existsSync.mockImplementation(() => false);
destination(mock.userRequest, mock.file, callback);
expect(existsSync).toHaveBeenCalled();
expect(mkdirSync).toHaveBeenCalled();
});
it('should return the destination', () => {
destination(mock.userRequest, mock.file, callback);
expect(mkdirSync).not.toHaveBeenCalled();
expect(callback).toHaveBeenCalledWith(null, 'upload/profile/test-user');
});
});
describe('filename', () => {
it('should require a user', () => {
filename(mock.req, mock.file, callback);
expect(callback).toHaveBeenCalled();
const [error, name] = callback.mock.calls[0];
expect(error).toBeDefined();
expect(name).toBeUndefined();
});
it('should return the filename', () => {
filename(mock.userRequest, mock.file, callback);
expect(mkdirSync).not.toHaveBeenCalled();
expect(callback).toHaveBeenCalledWith(null, 'test-user.jpg');
});
it('should sanitize the filename', () => {
filename(mock.userRequest, { ...mock.file, originalname: 'test.j\u0000pg' }, callback);
expect(callback).toHaveBeenCalledWith(null, 'test-user.jpg');
});
});
});

View File

@ -1,61 +0,0 @@
import { AuthUserDto, StorageCore, StorageFolder } from '@app/domain';
import { BadRequestException, UnauthorizedException } from '@nestjs/common';
import { MulterOptions } from '@nestjs/platform-express/multer/interfaces/multer-options.interface';
import { existsSync, mkdirSync } from 'fs';
import { diskStorage } from 'multer';
import { extname } from 'path';
import sanitize from 'sanitize-filename';
import { AuthRequest } from '../app.guard';
import { patchFormData } from '../app.utils';
export const profileImageUploadOption: MulterOptions = {
fileFilter,
storage: diskStorage({
destination,
filename,
}),
};
export const multerUtils = { fileFilter, filename, destination };
const storageCore = new StorageCore();
function fileFilter(req: AuthRequest, file: any, cb: any) {
if (!req.user) {
return cb(new UnauthorizedException());
}
if (file.mimetype.match(/\/(jpg|jpeg|png|heic|heif|dng|webp|avif)$/)) {
cb(null, true);
} else {
cb(new BadRequestException(`Unsupported file type ${extname(file.originalname)}`), false);
}
}
function destination(req: AuthRequest, file: Express.Multer.File, cb: any) {
if (!req.user) {
return cb(new UnauthorizedException());
}
const user = req.user as AuthUserDto;
const profileImageLocation = storageCore.getFolderLocation(StorageFolder.PROFILE, user.id);
if (!existsSync(profileImageLocation)) {
mkdirSync(profileImageLocation, { recursive: true });
}
cb(null, profileImageLocation);
}
function filename(req: AuthRequest, file: Express.Multer.File, cb: any) {
if (!req.user) {
return cb(new UnauthorizedException());
}
file.originalname = patchFormData(file.originalname);
const userId = req.user.id;
const fileName = `${userId}${extname(file.originalname)}`;
cb(null, sanitize(String(fileName)));
}

View File

@ -25,15 +25,14 @@ import {
UploadedFile,
UseInterceptors,
} from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { ApiBody, ApiConsumes, ApiTags } from '@nestjs/swagger';
import { Response as Res } from 'express';
import { AdminRoute, Authenticated, AuthUser, PublicRoute } from '../app.guard';
import { FileUploadInterceptor, Route } from '../app.interceptor';
import { UseValidation } from '../app.utils';
import { profileImageUploadOption } from '../config/profile-image-upload.config';
@ApiTags('User')
@Controller('user')
@Controller(Route.USER)
@Authenticated()
@UseValidation()
export class UserController {
@ -83,12 +82,9 @@ export class UserController {
return this.service.updateUser(authUser, updateUserDto);
}
@UseInterceptors(FileInterceptor('file', profileImageUploadOption))
@UseInterceptors(FileUploadInterceptor)
@ApiConsumes('multipart/form-data')
@ApiBody({
description: 'A new avatar for the user',
type: CreateProfileImageDto,
})
@ApiBody({ description: 'A new avatar for the user', type: CreateProfileImageDto })
@Post('/profile-image')
createProfileImage(
@AuthUser() authUser: AuthUserDto,

View File

@ -1,11 +1,12 @@
import { ICryptoRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { compareSync, hash } from 'bcrypt';
import { createHash, randomBytes } from 'crypto';
import { createHash, randomBytes, randomUUID } from 'crypto';
import { createReadStream } from 'fs';
@Injectable()
export class CryptoRepository implements ICryptoRepository {
randomUUID = randomUUID;
randomBytes = randomBytes;
hashBcrypt = hash;

View File

@ -2,6 +2,7 @@ import { ICryptoRepository } from '@app/domain';
export const newCryptoRepositoryMock = (): jest.Mocked<ICryptoRepository> => {
return {
randomUUID: jest.fn().mockReturnValue('random-uuid'),
randomBytes: jest.fn().mockReturnValue(Buffer.from('random-bytes', 'utf8')),
compareBcrypt: jest.fn().mockReturnValue(true),
hashBcrypt: jest.fn().mockImplementation((input) => Promise.resolve(`${input} (hashed)`)),