diff --git a/app/models/user.js b/app/models/user.js index a06a232a1..bdc3c13f4 100644 --- a/app/models/user.js +++ b/app/models/user.js @@ -21,7 +21,7 @@ import { nodeDirname } from '../support/node-dirname'; import { EventService } from '../support/EventService'; import { userCooldownStart, userDataDeletionStart, userPausedStart } from '../jobs/user-gone'; import { allExternalProviders } from '../support/ExtAuth'; -import { spawnAsync } from '../support/spawn-async'; +import { runImageMagick } from '../support/image-magick'; import { validate as validateUserPrefs } from './user-prefs'; @@ -1040,7 +1040,7 @@ export function addModel(dbAdapter) { const retinaSize = size * 2; const tmpPictureFile = join(tmpdir(), `resized-${uuid}-${size}`); - await spawnAsync('convert', [ + await runImageMagick('convert', [ path, '-auto-orient', '-resize', diff --git a/app/setup/environment.js b/app/setup/environment.js index 29e4d94f3..a3d9b9d5f 100644 --- a/app/setup/environment.js +++ b/app/setup/environment.js @@ -38,15 +38,22 @@ process.env.MONITOR_PREFIX = config.monitorPrefix; const checkIfMediaDirectoriesExist = async () => { let gotErrors = false; - const attConf = currentConfig().attachments; - - const attachmentsDir = attConf.storage.rootDir + attConf.path; - - try { - await access(attachmentsDir, constants.W_OK); - } catch { - gotErrors = true; - log(`Attachments dir does not exist: ${attachmentsDir}`); + const { attachments, profilePictures } = currentConfig(); + const dirs = [ + ['Attachments', attachments], + ['Profile pictures', profilePictures], + ].filter(([, mediaConfig]) => mediaConfig.storage.type === 'fs'); + + for (const [name, mediaConfig] of dirs) { + const dir = mediaConfig.storage.rootDir + mediaConfig.path; + + try { + // eslint-disable-next-line no-await-in-loop + await access(dir, constants.W_OK); + } catch { + gotErrors = true; + log(`${name} dir does not exist: ${dir}`); + } } if (gotErrors) { diff --git a/app/support/image-magick.ts b/app/support/image-magick.ts new file mode 100644 index 000000000..2bb1eab21 --- /dev/null +++ b/app/support/image-magick.ts @@ -0,0 +1,66 @@ +import config from 'config'; +import createDebug from 'debug'; + +import { spawnAsync, type SpawnAsyncArgs, type SpawnAsyncOptions } from './spawn-async'; + +type ImageMagickCommand = 'convert' | 'identify'; +type Runner = { command: string | null; useMagickCli: boolean }; + +const debug = createDebug('freefeed:image-magick'); + +let detectedRunner: Promise | null = null; + +export async function runImageMagick( + command: ImageMagickCommand, + args: SpawnAsyncArgs, + options: SpawnAsyncOptions & { binary: true }, +): Promise<{ stdout: Buffer; stderr: string }>; +export async function runImageMagick( + command: ImageMagickCommand, + args: SpawnAsyncArgs, + options?: SpawnAsyncOptions, +): Promise<{ stdout: string; stderr: string }>; +export async function runImageMagick( + command: ImageMagickCommand, + args: SpawnAsyncArgs, + options: SpawnAsyncOptions = {}, +): Promise<{ stdout: string | Buffer; stderr: string }> { + const runner = await getRunner(); + const runnerArgs = runner.useMagickCli ? magickArgs(command, args) : args; + + debug('run %s %o', runner.command ?? command, runnerArgs.flat()); + return await spawnAsync(runner.command ?? command, runnerArgs, options); +} + +function getRunner(): Promise { + detectedRunner ??= detectRunner(); + return detectedRunner; +} + +async function detectRunner(): Promise { + const { command } = config.imageMagick; + + if (command === 'legacy') { + debug('using legacy ImageMagick commands'); + return { command: null, useMagickCli: false }; + } + + if (command !== 'auto') { + debug('using configured ImageMagick command: %s', command); + return { command, useMagickCli: true }; + } + + try { + await spawnAsync('magick', ['-version']); + debug('using ImageMagick 7 magick CLI'); + return { command: 'magick', useMagickCli: true }; + } catch (e) { + const message = e instanceof Error ? e.message : e; + debug('magick CLI is not available, using legacy commands: %s', message); + return { command: null, useMagickCli: false }; + } +} + +function magickArgs(command: ImageMagickCommand, args: SpawnAsyncArgs): SpawnAsyncArgs { + return command === 'identify' ? ['identify', ...args] : args; +} diff --git a/app/support/media-files/detect.ts b/app/support/media-files/detect.ts index ae57ce95b..ed8cfff04 100644 --- a/app/support/media-files/detect.ts +++ b/app/support/media-files/detect.ts @@ -1,5 +1,6 @@ import { open } from 'fs/promises'; +import { runImageMagick } from '../image-magick'; import { spawnAsync } from '../spawn-async'; import { @@ -22,7 +23,7 @@ export async function detectMediaType( if (probablyImage) { // Identify using ImageMagick try { - const out = await spawnAsync('identify', [ + const out = await runImageMagick('identify', [ ['-format', '%m %W %H %[orientation] %n|'], `${localFilePath}[0,1]`, // Select only up to 2 first frames to reduce the memory usage ]); diff --git a/app/support/media-files/process.ts b/app/support/media-files/process.ts index 452be36a0..8f185e34a 100644 --- a/app/support/media-files/process.ts +++ b/app/support/media-files/process.ts @@ -4,6 +4,7 @@ import { lookup as mimeLookup } from 'mime-types'; import { exiftool } from 'exiftool-vendored'; import createDebug from 'debug'; +import { runImageMagick } from '../image-magick'; import { spawnAsync, type SpawnAsyncArgs } from '../spawn-async'; import { currentConfig } from '../app-async-context'; import { ContentTooLargeException } from '../exceptions'; @@ -247,7 +248,7 @@ async function processImage( return; } - await spawnAsync('convert', [ + await runImageMagick('convert', [ `${localFilePath}[0]`, // Adding [0] for the case of animated or multi-page images '-auto-orient', ['-resize', `${width}!x${height}!`], diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 6ec85cabb..b10bdf5dc 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -6,6 +6,9 @@ "host": "FRFS_REDIS_HOST", "port": { "__name": "FRFS_REDIS_PORT", "__format": "json" } }, + "imageMagick": { + "command": "FRFS_IMAGEMAGICK_COMMAND" + }, "postgres": { "connection": { "host": "FRFS_PG_HOST", diff --git a/config/default.js b/config/default.js index 08ed4b8c1..7f2187a18 100644 --- a/config/default.js +++ b/config/default.js @@ -209,6 +209,13 @@ config.profilePictures = { path: 'profilepics/', // must have trailing slash }; +config.imageMagick = { + // 'auto' prefers ImageMagick 7 'magick' CLI and falls back to ImageMagick 6 + // legacy 'convert'/'identify' commands. Set to 'legacy' to force ImageMagick 6 + // commands, or to 'magick' / a path to magick to force ImageMagick 7 CLI. + command: 'auto', +}; + config.mailer = { useSMTPTransport: false, transport: defer((cfg) => (cfg.mailer.useSMTPTransport ? smtpTransport : stubTransport)), diff --git a/test/functional/attachments.js b/test/functional/attachments.js index e28357ff2..54f87948c 100644 --- a/test/functional/attachments.js +++ b/test/functional/attachments.js @@ -248,7 +248,7 @@ describe('Attachments', () => { // Test the v4 API response const resp1 = await performJSONRequest('GET', `/v4/attachments/${id}`); - expect(resp1.attachments, 'to equal', { + expect(resp1.attachments, 'to satisfy', { id, mediaType: 'audio', fileName: 'music.mp3', @@ -259,7 +259,9 @@ describe('Attachments', () => { 'dc:creator': 'Piermic', 'dc:relation.isPartOf': 'Wikimedia', }, - duration: 24.032653, + // Different FFmpeg versions may report slightly different durations, so + // we allow a small delta + duration: expect.it('to be close to', 24.032653, 0.1), createdAt: attObj.createdAt.toISOString(), updatedAt: attObj.updatedAt.toISOString(), createdBy: luna.user.id, diff --git a/test/functional/bookmarklet.js b/test/functional/bookmarklet.js index 8ed7dfc9c..f86d29445 100644 --- a/test/functional/bookmarklet.js +++ b/test/functional/bookmarklet.js @@ -125,7 +125,7 @@ describe('BookmarkletController', () => { } else if (url === '/big-body.jpg') { ctx.status = 200; ctx.response.type = 'image/jpeg'; - ctx.body = getStreamOfLength(imageSizeLimit * 2); + ctx.body = getStreamOfLength(imageSizeLimit + 1024); } else if (url === '/im%C3%A5ge.png') { ctx.status = 200; ctx.response.type = 'image/png'; @@ -250,7 +250,7 @@ async function callBookmarklet(author, body) { return respBody; } -function getStreamOfLength(length, chunk = Buffer.alloc(1024)) { +function getStreamOfLength(length, chunk = Buffer.alloc(1024 * 1024)) { let bytesLeft = length; return new Readable({ read() { @@ -258,7 +258,7 @@ function getStreamOfLength(length, chunk = Buffer.alloc(1024)) { this.push(chunk); bytesLeft -= chunk.length; } else if (bytesLeft > 0) { - this.push(chunk.slice(0, bytesLeft)); + this.push(chunk.subarray(0, bytesLeft)); bytesLeft = 0; } else { this.push(null); diff --git a/test/functional/groups.js b/test/functional/groups.js index a0837a844..4adb6e648 100644 --- a/test/functional/groups.js +++ b/test/functional/groups.js @@ -1,8 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions */ /* global $pg_database */ import request from 'superagent'; -import { mkdirp } from 'mkdirp'; -import config from 'config'; import cleanDB from '../dbCleaner'; import { getSingleton } from '../../app/app'; @@ -349,7 +347,6 @@ describe('GroupsController', () => { beforeEach(async () => { context = await funcTestHelper.createUserAsync('Luna', 'password'); - await mkdirp(config.profilePictures.storage.rootDir + config.profilePictures.path); await funcTestHelper.createGroupAsync(context, 'pepyatka-dev', 'Pepyatka Developers'); }); diff --git a/test/functional/users.js b/test/functional/users.js index 86b33d922..0ea57f1b1 100755 --- a/test/functional/users.js +++ b/test/functional/users.js @@ -2,7 +2,6 @@ /* global $pg_database, $should */ import jwt from 'jsonwebtoken'; import * as _ from 'lodash-es'; -import { mkdirp } from 'mkdirp'; import request from 'superagent'; import expect from 'unexpected'; import config from 'config'; @@ -1388,10 +1387,7 @@ describe('UsersController', () => { let authToken; beforeEach(async () => { - const [user] = await Promise.all([ - funcTestHelper.createUserAsync('Luna', 'password'), - mkdirp(config.profilePictures.storage.rootDir + config.profilePictures.path), - ]); + const user = await funcTestHelper.createUserAsync('Luna', 'password'); ({ authToken } = user); }); diff --git a/test/integration/models/attachment-orientation.js b/test/integration/models/attachment-orientation.js index 4cbf2558f..282aa6fde 100644 --- a/test/integration/models/attachment-orientation.js +++ b/test/integration/models/attachment-orientation.js @@ -8,7 +8,7 @@ import expect from 'unexpected'; import cleanDB from '../../dbCleaner'; import { User, Attachment } from '../../../app/models'; -import { spawnAsync } from '../../../app/support/spawn-async'; +import { runImageMagick } from '../../../app/support/image-magick'; const orientationNames = [ 'Undefined', // No orientation tag @@ -67,7 +67,7 @@ describe('Orientation', () => { }); async function getOrientation(filename) { - const out = await spawnAsync('identify', ['-format', '%[orientation]', filename]); + const out = await runImageMagick('identify', ['-format', '%[orientation]', filename]); return out.stdout; } @@ -76,7 +76,7 @@ async function getOrientation(filename) { * orientation tag when it is not zero. */ async function createTestImage(filename, orientation) { - await spawnAsync('convert', [ + await runImageMagick('convert', [ ['-size', '200x300'], 'xc:#000000', ['-fill', '#ffffff'], @@ -115,7 +115,7 @@ function patternForOrientation(orientation) { } async function expectOrientation(filePath, orientation) { - const { stdout: buffer } = await spawnAsync( + const { stdout: buffer } = await runImageMagick( 'convert', [filePath, '-filter', 'Point', '-resize', '3x3', 'gray:-'], { binary: true }, diff --git a/test/integration/models/attachment.js b/test/integration/models/attachment.js index fe785fcb5..78d387f84 100755 --- a/test/integration/models/attachment.js +++ b/test/integration/models/attachment.js @@ -5,7 +5,6 @@ import { tmpdir } from 'os'; import { v4 as createUuid } from 'uuid'; import { before, beforeEach, describe, it } from 'mocha'; import expect from 'unexpected'; -import { mkdirp } from 'mkdirp'; import { lookup as mimeLookup } from 'mime-types'; import cleanDB from '../../dbCleaner'; @@ -13,7 +12,7 @@ import { dbAdapter, Attachment } from '../../../app/models'; import { currentConfig } from '../../../app/support/app-async-context'; import { createUser } from '../helpers/users'; import { createPost } from '../helpers/posts-and-comments'; -import { spawnAsync } from '../../../app/support/spawn-async'; +import { runImageMagick } from '../../../app/support/image-magick'; import { withModifiedConfig } from '../../helpers/with-modified-config'; import { initJobProcessing } from '../../../app/jobs'; import { ATTACHMENT_PREPARE_VIDEO } from '../../../app/jobs/attachment-prepare-video'; @@ -35,11 +34,6 @@ describe('Attachments', () => { // Create user user = await createUser('luna'); jobManager = await initJobProcessing(); - - const attConf = currentConfig().attachments; - - // Create directories for attachments - await mkdirp(attConf.storage.rootDir + attConf.path); }); beforeEach(() => fakeS3Storage.clear()); @@ -112,7 +106,7 @@ describe('Attachments', () => { currentConfig().attachments.storage.rootDir, att.getRelFilePath('p2', 'webp'), ); - const out = await spawnAsync('identify', ['-format', '%w %h %[orientation]', originalFile]); + const out = await runImageMagick('identify', ['-format', '%w %h %[orientation]', originalFile]); expect(out.stdout, 'to equal', '300 900 Undefined'); }); @@ -123,7 +117,7 @@ describe('Attachments', () => { // original colors { const originalFile = join(rootDir, att.getRelFilePath('', att.fileExtension)); - const { stdout: buffer } = await spawnAsync( + const { stdout: buffer } = await runImageMagick( 'convert', [originalFile, '-resize', '1x1!', '-colorspace', 'sRGB', '-depth', '8', 'rgb:-'], { binary: true }, @@ -138,7 +132,7 @@ describe('Attachments', () => { // preview colors { const previewFile = join(rootDir, att.getRelFilePath('p1', 'webp')); - const { stdout: buffer } = await spawnAsync( + const { stdout: buffer } = await runImageMagick( 'convert', [previewFile, '-resize', '1x1!', '-colorspace', 'sRGB', '-depth', '8', 'rgb:-'], { binary: true }, diff --git a/test/integration/models/media-files.js b/test/integration/models/media-files.js index 155078752..a0dab5161 100644 --- a/test/integration/models/media-files.js +++ b/test/integration/models/media-files.js @@ -16,7 +16,15 @@ describe('Media files', () => { const { file: fileName, info } = file; it(`should detect media info for ${fileName}`, async () => { const detected = await detectMediaType(join(samplesDir, fileName), fileName); - expect(info, 'to equal', detected); + + if (info.duration) { + // Different FFmpeg versions may report slightly different durations, so + // we allow a small delta + const d = info.duration; + info.duration = expect.it('to be close to', d, 0.1); + } + + expect(detected, 'to satisfy', info); }); } }); diff --git a/test/testHelper.js b/test/testHelper.js index 3f6bdfe43..acba10e8d 100644 --- a/test/testHelper.js +++ b/test/testHelper.js @@ -1,4 +1,6 @@ import { should } from 'chai'; +import config from 'config'; +import { mkdirp } from 'mkdirp'; import redisDb from '../app/setup/database'; import * as postgresDb from '../app/setup/postgres'; @@ -12,3 +14,13 @@ global.$pg_database = global.$postgres.connect(); // Show stack not only for errors, but also for warnings process.on('warning', (e) => console.warn(e.stack)); + +export const mochaHooks = { + async beforeAll() { + const mediaDirs = [config.attachments, config.profilePictures] + .filter((mediaConfig) => mediaConfig.storage.type === 'fs') + .map((mediaConfig) => mkdirp(mediaConfig.storage.rootDir + mediaConfig.path)); + + await Promise.all(mediaDirs); + }, +}; diff --git a/types/config.d.ts b/types/config.d.ts index f2c81c02b..4d6c74c45 100644 --- a/types/config.d.ts +++ b/types/config.d.ts @@ -224,6 +224,13 @@ declare module 'config' { hashtagStats: { refreshInterval: number; }; + + imageMagick: { + // 'auto' prefers ImageMagick 7 'magick' CLI and falls back to ImageMagick 6 + // legacy 'convert'/'identify' commands. Set to 'legacy' to force ImageMagick 6 + // commands, or to 'magick' / a path to magick to force ImageMagick 7 CLI. + command: 'auto' | 'legacy' | string; + }; }; export type TranslationLimits = {