diff --git a/server/e2e/jobs/specs/library-watcher.e2e-spec.ts b/server/e2e/jobs/specs/library-watcher.e2e-spec.ts index f9c50a411f..cb7dd5f894 100644 --- a/server/e2e/jobs/specs/library-watcher.e2e-spec.ts +++ b/server/e2e/jobs/specs/library-watcher.e2e-spec.ts @@ -1,8 +1,6 @@ import { LibraryResponseDto, LibraryService, LoginResponseDto } from '@app/domain'; import { AssetType, LibraryType } from '@app/infra/entities'; -import * as fs from 'fs/promises'; -import { api } from '../../client'; - +import fs from 'fs/promises'; import path from 'path'; import { IMMICH_TEST_ASSET_PATH, @@ -11,6 +9,7 @@ import { testApp, waitForEvent, } from '../../../src/test-utils/utils'; +import { api } from '../../client'; describe(`Library watcher (e2e)`, () => { let server: any; @@ -45,11 +44,9 @@ describe(`Library watcher (e2e)`, () => { }); describe('Event handling', () => { - let library: LibraryResponseDto; - describe('Single import path', () => { beforeEach(async () => { - library = await api.libraryApi.create(server, admin.accessToken, { + await api.libraryApi.create(server, admin.accessToken, { type: LibraryType.EXTERNAL, importPaths: [`${IMMICH_TEST_ASSET_TEMP_PATH}`], }); @@ -139,7 +136,7 @@ describe(`Library watcher (e2e)`, () => { await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir2`, { recursive: true }); await fs.mkdir(`${IMMICH_TEST_ASSET_TEMP_PATH}/dir3`, { recursive: true }); - library = await api.libraryApi.create(server, admin.accessToken, { + await api.libraryApi.create(server, admin.accessToken, { type: LibraryType.EXTERNAL, importPaths: [ `${IMMICH_TEST_ASSET_TEMP_PATH}/dir1`, diff --git a/server/package-lock.json b/server/package-lock.json index e6e5a5aaf0..3a449e4df4 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -28,6 +28,7 @@ "axios": "^1.5.0", "bcrypt": "^5.1.1", "bullmq": "^4.8.0", + "chokidar": "^3.5.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -78,7 +79,6 @@ "@types/ua-parser-js": "^0.7.36", "@typescript-eslint/eslint-plugin": "^6.4.1", "@typescript-eslint/parser": "^6.4.1", - "chokidar": "^3.5.3", "dotenv": "^16.3.1", "eslint": "^8.48.0", "eslint-config-prettier": "^9.0.0", @@ -3981,7 +3981,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -3994,7 +3993,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -4370,7 +4368,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, "engines": { "node": ">=8" } @@ -4434,7 +4431,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -4720,7 +4716,6 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, "funding": [ { "type": "individual", @@ -6230,7 +6225,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -6530,7 +6524,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "optional": true, "os": [ @@ -6702,7 +6695,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "dependencies": { "is-glob": "^4.0.1" }, @@ -7115,7 +7107,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -7139,7 +7130,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7165,7 +7155,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "dependencies": { "is-extglob": "^2.1.1" }, @@ -7185,7 +7174,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -9791,7 +9779,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -9803,7 +9790,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "engines": { "node": ">=8.6" }, @@ -11283,7 +11269,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "dependencies": { "is-number": "^7.0.0" }, @@ -15043,7 +15028,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -15052,8 +15036,7 @@ "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" } } }, @@ -15355,8 +15338,7 @@ "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, "bl": { "version": "4.1.0", @@ -15415,7 +15397,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "dev": true, "requires": { "fill-range": "^7.0.1" } @@ -15601,7 +15582,6 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "dev": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -16743,7 +16723,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "dev": true, "requires": { "to-regex-range": "^5.0.1" } @@ -16969,7 +16948,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "optional": true }, "function-bind": { @@ -17101,7 +17079,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, "requires": { "is-glob": "^4.0.1" } @@ -17372,7 +17349,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "requires": { "binary-extensions": "^2.0.0" } @@ -17389,8 +17365,7 @@ "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==" }, "is-fullwidth-code-point": { "version": "3.0.0", @@ -17407,7 +17382,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "requires": { "is-extglob": "^2.1.1" } @@ -17420,8 +17394,7 @@ "is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "is-path-inside": { "version": "3.0.3", @@ -19405,7 +19378,6 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, "requires": { "picomatch": "^2.2.1" }, @@ -19413,8 +19385,7 @@ "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==" } } }, @@ -20565,7 +20536,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "requires": { "is-number": "^7.0.0" } diff --git a/server/package.json b/server/package.json index e93a8f9d9b..e477e33a5f 100644 --- a/server/package.json +++ b/server/package.json @@ -53,6 +53,7 @@ "axios": "^1.5.0", "bcrypt": "^5.1.1", "bullmq": "^4.8.0", + "chokidar": "^3.5.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -103,7 +104,6 @@ "@types/ua-parser-js": "^0.7.36", "@typescript-eslint/eslint-plugin": "^6.4.1", "@typescript-eslint/parser": "^6.4.1", - "chokidar": "^3.5.3", "dotenv": "^16.3.1", "eslint": "^8.48.0", "eslint-config-prettier": "^9.0.0", diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index b79bb309c8..c064cd1b13 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -1,19 +1,17 @@ import { AssetType, LibraryType } from '@app/infra/entities'; +import { ImmichLogger } from '@app/infra/logger'; import { BadRequestException, Inject, Injectable } from '@nestjs/common'; -import picomatch from 'picomatch'; - +import { EventEmitter } from 'events'; import { R_OK } from 'node:constants'; import { Stats } from 'node:fs'; import path from 'node:path'; import { basename, parse } from 'path'; +import picomatch from 'picomatch'; import { AccessCore, Permission } from '../access'; import { AuthDto } from '../auth'; import { mimeTypes } from '../domain.constant'; import { usePagination, validateCronExpression } from '../domain.util'; import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job'; - -import { ImmichLogger } from '@app/infra/logger'; -import { EventEmitter } from 'events'; import { IAccessRepository, IAssetRepository, @@ -59,31 +57,33 @@ export class LibraryService extends EventEmitter { this.access = AccessCore.create(accessRepository); this.configCore = SystemConfigCore.create(configRepository); this.configCore.addValidator((config) => { - if (!validateCronExpression(config.library.scan.cronExpression)) { - throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`); + const { scan } = config.library; + if (!validateCronExpression(scan.cronExpression)) { + throw new Error(`Invalid cron expression ${scan.cronExpression}`); } }); } async init() { const config = await this.configCore.getConfig(); - this.watchLibraries = config.library.watch.enabled; + const { watch, scan } = config.library; + this.watchLibraries = watch.enabled; this.jobRepository.addCronJob( 'libraryScan', - config.library.scan.cronExpression, + scan.cronExpression, () => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }), - config.library.scan.enabled, + scan.enabled, ); if (this.watchLibraries) { await this.watchAll(); } - this.configCore.config$.subscribe(async (config) => { - this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled); + this.configCore.config$.subscribe(async ({ library }) => { + this.jobRepository.updateCronJob('libraryScan', library.scan.cronExpression, library.scan.enabled); - if (config.library.watch.enabled !== this.watchLibraries) { - this.watchLibraries = config.library.watch.enabled; + if (library.watch.enabled !== this.watchLibraries) { + this.watchLibraries = library.watch.enabled; if (this.watchLibraries) { await this.watchAll(); } else { @@ -116,15 +116,14 @@ export class LibraryService extends EventEmitter { }); const config = await this.configCore.getConfig(); + const { usePolling, interval } = config.library.watch; - this.logger.debug( - `Settings for watcher: usePolling: ${config.library.watch.usePolling}, interval: ${config.library.watch.interval}`, - ); + this.logger.debug(`Settings for watcher: usePolling: ${usePolling}, interval: ${interval}`); const watcher = this.storageRepository.watch(library.importPaths, { - usePolling: config.library.watch.usePolling, - interval: config.library.watch.interval, - binaryInterval: config.library.watch.interval, + usePolling, + interval, + binaryInterval: interval, ignoreInitial: true, }); diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index a6adf50ddd..95d25ab26b 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -53,9 +53,6 @@ export enum SystemConfigKey { LIBRARY_WATCH_ENABLED = 'library.watch.enabled', LIBRARY_WATCH_USE_POLLING = 'library.watch.usePolling', LIBRARY_WATCH_INTERVAL = 'library.watch.interval', - LIBRARY_WATCH_BINARY_INTERVAL = 'library.watch.binaryInterval', - LIBRARY_WATCH_WRITE_STABILITY_THRESHOLD = 'library.watch.awaitWriteFinish.stabilityThreshold', - LIBRARY_WATCH_WRITE_POLL_INTERVAL = 'library.watch.awaitWriteFinish.pollInterval', LOGGING_ENABLED = 'logging.enabled', LOGGING_LEVEL = 'logging.level',