From f59430d5f36d63983357f55db3c7cf2e3b3fd73a Mon Sep 17 00:00:00 2001 From: Alban Mouton Date: Thu, 7 May 2026 16:09:30 +0200 Subject: [PATCH 01/36] feat: work on registry integration --- api/config/custom-environment-variables.mjs | 4 +- api/config/default.mjs | 15 +- api/config/development.mjs | 5 +- api/config/type/schema.json | 17 +- api/doc/plugin/post-req/index.ts | 1 - api/doc/plugin/post-req/schema.js | 21 - api/package.json | 1 + api/src/admin/status.ts | 12 +- api/src/app.ts | 4 - api/src/config.ts | 18 +- api/src/misc/routers/test-env.ts | 9 +- api/src/misc/utils/api-docs.ts | 321 +-------- api/src/plugins-registry/router.ts | 75 -- api/src/plugins/router.ts | 308 -------- api/src/processings/router.ts | 113 ++- api/src/runs/service.ts | 2 +- api/src/server.ts | 2 +- api/types/index.ts | 1 - api/types/plugin/index.ts | 1 - api/types/plugin/schema.js | 103 --- api/types/processing/schema.js | 7 +- dev/init-env.sh | 1 + dev/resources/nginx.conf.template | 4 + docker-compose.yml | 25 + docs/architecture/v6-registry-integration.md | 333 +++++++++ package-lock.json | 131 +++- shared/plugin-id.ts | 15 + tests/features/plugins/install.api.spec.ts | 137 ---- tests/features/plugins/registry.api.spec.ts | 21 - .../processings/lifecycle.api.spec.ts | 51 +- .../processings/permissions.api.spec.ts | 37 +- tests/features/ui/layout.e2e.spec.ts | 16 +- tests/support/axios.ts | 4 + tests/support/registry.ts | 108 +++ .../processing/processing-actions.vue | 10 +- .../components/processing/processing-card.vue | 6 +- ui/src/components/processings-actions.vue | 28 +- ui/src/composables/use-plugin-fetch.ts | 36 +- ui/src/pages/admin/index.vue | 9 - ui/src/pages/admin/plugins.vue | 675 ------------------ ui/src/pages/dev.vue | 7 - ui/src/pages/processings/[id]/index.vue | 42 +- ui/src/pages/processings/index.vue | 2 +- ui/src/pages/processings/new.vue | 168 +++-- .../6.0.0/01-publish-plugins-to-registry.ts | 201 ++++++ upgrade/6.0.0/02-rewrite-processing-plugin.ts | 80 +++ .../config/custom-environment-variables.mjs | 6 +- worker/config/default.mjs | 14 +- worker/config/development.mjs | 5 +- worker/config/type/schema.json | 17 +- worker/package.json | 6 +- worker/src/config.ts | 17 +- worker/src/task/axios.ts | 2 +- worker/src/task/task.ts | 53 +- worker/src/worker.ts | 2 +- 55 files changed, 1396 insertions(+), 1913 deletions(-) delete mode 100644 api/doc/plugin/post-req/index.ts delete mode 100644 api/doc/plugin/post-req/schema.js delete mode 100644 api/src/plugins-registry/router.ts delete mode 100644 api/src/plugins/router.ts delete mode 100644 api/types/plugin/index.ts delete mode 100644 api/types/plugin/schema.js create mode 100644 docs/architecture/v6-registry-integration.md create mode 100644 shared/plugin-id.ts delete mode 100644 tests/features/plugins/install.api.spec.ts delete mode 100644 tests/features/plugins/registry.api.spec.ts create mode 100644 tests/support/registry.ts delete mode 100644 ui/src/pages/admin/index.vue delete mode 100644 ui/src/pages/admin/plugins.vue create mode 100644 upgrade/6.0.0/01-publish-plugins-to-registry.ts create mode 100644 upgrade/6.0.0/02-rewrite-processing-plugin.ts diff --git a/api/config/custom-environment-variables.mjs b/api/config/custom-environment-variables.mjs index 37476549..b853573b 100644 --- a/api/config/custom-environment-variables.mjs +++ b/api/config/custom-environment-variables.mjs @@ -13,10 +13,12 @@ export default { port: 'PORT', privateDirectoryUrl: 'PRIVATE_DIRECTORY_URL', privateEventsUrl: 'PRIVATE_EVENTS_URL', + privateRegistryUrl: 'PRIVATE_REGISTRY_URL', secretKeys: { limits: 'SECRET_LIMITS', events: 'SECRET_EVENTS', - identities: 'SECRET_IDENTITIES' + identities: 'SECRET_IDENTITIES', + registry: 'SECRET_REGISTRY' }, observer: { active: 'OBSERVER_ACTIVE', diff --git a/api/config/default.mjs b/api/config/default.mjs index 3f7f83d1..be441fae 100644 --- a/api/config/default.mjs +++ b/api/config/default.mjs @@ -1,7 +1,10 @@ export default { cipherPassword: undefined, - dataDir: '/app/data', - tmpDir: null, // will be dataDir + '/tmp' if null + // Optional. When set, the legacy plugins volume at /plugins is read + // by the v6.0 boot migration. Drops with v7.0. + dataDir: null, + // Defaults to /tmp when dataDir is set, else /data-fair-processings. + tmpDir: null, defaultLimits: { // Maximum time spent running processings // -1 for unlimited storage @@ -10,10 +13,16 @@ export default { pluginCategories: ['Essentiels', 'Mes plugins', 'Données de références', 'Tests'], privateDirectoryUrl: 'http://simple-directory:8080', privateEventsUrl: undefined, + // Internal URL the API uses for server-to-server calls to registry. The UI + // talks to /registry on the same domain (no public URL is configurable). + privateRegistryUrl: 'http://registry:8080', secretKeys: { limits: null, events: undefined, - identities: undefined + identities: undefined, + // x-secret-key shared with registry. Required for save-time validation, + // prepare-hook downloads, and the v6.0 first-boot migration. + registry: undefined }, mongoUrl: 'mongodb://localhost:27017/data-fair-processings', port: 8080, diff --git a/api/config/development.mjs b/api/config/development.mjs index d89fdd38..0958cc36 100644 --- a/api/config/development.mjs +++ b/api/config/development.mjs @@ -2,6 +2,7 @@ const apiPort = parseInt(process.env.DEV_API_PORT ?? '8082') const mongoPort = process.env.MONGO_PORT ?? '27017' const sdPort = process.env.SD_PORT ?? '8080' const eventsPort = process.env.EVENTS_PORT ?? '8083' +const registryPort = process.env.REGISTRY_PORT ?? '8085' const observerPort = parseInt(process.env.DEV_API_OBSERVER_PORT ?? '9092') export default { @@ -14,8 +15,10 @@ export default { port: apiPort, privateDirectoryUrl: `http://localhost:${sdPort}`, privateEventsUrl: `http://localhost:${eventsPort}`, + privateRegistryUrl: `http://localhost:${registryPort}/registry`, secretKeys: { identities: 'secret-identities', - events: 'secret-events' + events: 'secret-events', + registry: 'secret-registry-internal' } } diff --git a/api/config/type/schema.json b/api/config/type/schema.json index 8c913248..9e8c1f41 100644 --- a/api/config/type/schema.json +++ b/api/config/type/schema.json @@ -12,8 +12,8 @@ "additionalProperties": false, "required": [ "cipherPassword", - "dataDir", "privateDirectoryUrl", + "privateRegistryUrl", "mongoUrl", "pluginCategories", "port", @@ -25,7 +25,8 @@ "type": "string" }, "dataDir": { - "type": "string" + "type": ["string", "null"], + "description": "Optional. When set, the legacy plugins volume at /plugins is read by the v6.0 boot migration. Drops with v7.0." }, "tmpDir": { "type": "string" @@ -57,8 +58,14 @@ "privateEventsUrl": { "type": "string" }, + "privateRegistryUrl": { + "type": "string", + "pattern": "^https?://", + "description": "Internal URL used by the API to call registry server-to-server. The UI hits /registry on the same domain." + }, "secretKeys": { "type": "object", + "required": ["registry"], "properties": { "limits": { "type": [ @@ -71,6 +78,10 @@ }, "identities": { "type": "string" + }, + "registry": { + "type": "string", + "description": "x-secret-key shared with registry. Required for save-time validation, prepare-hook downloads, and the v6.0 first-boot migration." } } }, @@ -97,4 +108,4 @@ "get": {}, "has": {} } -} \ No newline at end of file +} diff --git a/api/doc/plugin/post-req/index.ts b/api/doc/plugin/post-req/index.ts deleted file mode 100644 index f5d615bd..00000000 --- a/api/doc/plugin/post-req/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './.type/index.js' diff --git a/api/doc/plugin/post-req/schema.js b/api/doc/plugin/post-req/schema.js deleted file mode 100644 index f6c972d8..00000000 --- a/api/doc/plugin/post-req/schema.js +++ /dev/null @@ -1,21 +0,0 @@ -import jsonSchema from '@data-fair/lib-utils/json-schema.js' -import PluginSchema from '#types/plugin/schema.js' - -export default { - $id: 'https://github.com/data-fair/processings/plugin/post-req', - title: 'Post processing req', - 'x-exports': ['validate', 'types'], - type: 'object', - required: ['body'], - properties: { - body: - jsonSchema(PluginSchema) - .pickProperties(['distTag', 'name', 'version', 'description']) - .removeFromRequired([ - 'description' - ]) - .removeId() - .appendTitle(' post') - .schema - } -} diff --git a/api/package.json b/api/package.json index 71d0ee60..a3480388 100644 --- a/api/package.json +++ b/api/package.json @@ -23,6 +23,7 @@ "dependencies": { "@data-fair/lib-express": "^1.22.5", "@data-fair/lib-node": "^2.12.1", + "@data-fair/lib-node-registry": "file:../../registry/lib-node", "@data-fair/lib-utils": "^1.10.1", "@data-fair/processings-shared": "*", "ajv": "^8.17.1", diff --git a/api/src/admin/status.ts b/api/src/admin/status.ts index 2e03ddb1..fb962ea5 100644 --- a/api/src/admin/status.ts +++ b/api/src/admin/status.ts @@ -8,11 +8,13 @@ const volumeStatus = async () => { await fs.writeFile(`${config.dataDir}/check-access.txt`, 'ok') } -export const getStatus = async (req: Request) => - runHealthChecks(req, [ - { fn: mongoStatus, name: 'mongodb' }, - { fn: volumeStatus, name: 'data volume' } - ]) +export const getStatus = async (req: Request) => { + const checks: Array<{ fn: (req: Request) => Promise; name: string }> = [ + { fn: mongoStatus, name: 'mongodb' } + ] + if (config.dataDir) checks.push({ fn: volumeStatus, name: 'data volume' }) + return runHealthChecks(req, checks) +} // Helper functions const getSingleStatus = async (req: Request, fn: (req: Request) => Promise, name: string) => { diff --git a/api/src/app.ts b/api/src/app.ts index b9491735..dfe02a7b 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -3,8 +3,6 @@ import express from 'express' import { session, errorHandler, createSiteMiddleware, createSpaMiddleware, defaultNonceCSPDirectives } from '@data-fair/lib-express/index.js' import identitiesRouter from './misc/routers/identities.ts' import limitsRouter from './limits/router.ts' -import pluginsRegistryRouter from './plugins-registry/router.ts' -import pluginsRouter from './plugins/router.ts' import processingsRouter from './processings/router.ts' import runsRouter from './runs/router.ts' import adminRouter from './admin/router.ts' @@ -28,8 +26,6 @@ app.get('/api/v1/_ping', (req, res) => { }) app.use('/api/identities', identitiesRouter) -app.use('/api/v1/plugins-registry', pluginsRegistryRouter) -app.use('/api/v1/plugins', pluginsRouter) app.use('/api/v1/processings', processingsRouter) app.use('/api/v1/runs', runsRouter) app.use('/api/v1/limits', limitsRouter) diff --git a/api/src/config.ts b/api/src/config.ts index b134db30..902d1431 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -1,13 +1,27 @@ +import path from 'node:path' +import os from 'node:os' import type { ApiConfig } from '../config/type/index.ts' import { assertValid } from '../config/type/index.ts' import config from 'config' assertValid(config, { lang: 'en', name: 'config', internal: true }) -const apiConfig = config as unknown as ApiConfig +const rawConfig = config as unknown as ApiConfig + +if (!rawConfig.tmpDir) { + rawConfig.tmpDir = rawConfig.dataDir + ? path.join(rawConfig.dataDir, 'tmp') + : path.join(os.tmpdir(), 'data-fair-processings') +} + +const apiConfig = rawConfig as ApiConfig & { tmpDir: string } + +// cacheDir is always a subdirectory of tmpDir. +export const registryCacheDir = path.join(apiConfig.tmpDir, 'registry-cache') + export default apiConfig export const uiConfig = { - pluginCategories: apiConfig.pluginCategories, + pluginCategories: apiConfig.pluginCategories } export type UiConfig = typeof uiConfig diff --git a/api/src/misc/routers/test-env.ts b/api/src/misc/routers/test-env.ts index 6684d526..15490b33 100644 --- a/api/src/misc/routers/test-env.ts +++ b/api/src/misc/routers/test-env.ts @@ -93,11 +93,14 @@ router.post('/set-config', (req, res, next) => { } }) -// Wipe the installed plugins directory (used between test runs) +// Wipe the installed plugins directory (used between test runs). +// No-op when the legacy plugins volume isn't mounted (config.dataDir unset). router.delete('/plugins', async (req, res, next) => { try { - const pluginsDir = path.resolve(config.dataDir, 'plugins') - await fs.emptyDir(pluginsDir) + if (config.dataDir) { + const pluginsDir = path.resolve(config.dataDir, 'plugins') + await fs.emptyDir(pluginsDir) + } res.json({ ok: true }) } catch (err) { next(err) diff --git a/api/src/misc/utils/api-docs.ts b/api/src/misc/utils/api-docs.ts index 612a1af9..aa8ae49b 100644 --- a/api/src/misc/utils/api-docs.ts +++ b/api/src/misc/utils/api-docs.ts @@ -1,6 +1,5 @@ import jsonSchema from '@data-fair/lib-utils/json-schema.js' import { type Processing, resolvedSchema as ProcessingSchema } from '#types/processing/index.ts' -import { type Plugin, resolvedSchema as PluginSchema } from '#types/plugin/index.ts' import { readFileSync } from 'node:fs' import path from 'path' @@ -8,7 +7,10 @@ const packageJson = JSON.parse(readFileSync(path.resolve(import.meta.dirname, '. // CTRL + K CTRL + 4 to fold operations levels -export default (origin: string, options?: { processing?: Processing, plugin?: Plugin }) => { +// In v6 the API only needs the plugin's name, version and processingConfigSchema. +type ApiDocPlugin = { name: string, version: string, processingConfigSchema?: Record } + +export default (origin: string, options?: { processing?: Processing, plugin?: ApiDocPlugin }) => { if (options?.plugin?.processingConfigSchema) ProcessingSchema.properties.config = options?.plugin?.processingConfigSchema const doc: Record = { @@ -196,13 +198,13 @@ export default (origin: string, options?: { processing?: Processing, plugin?: Pl content: { 'application/json': { schema: jsonSchema(ProcessingSchema) - .pickProperties(['owner', 'plugin', 'title']) + .pickProperties(['owner', 'pluginId', 'title']) .removeFromRequired(['scheduling', '_id']) .removeId() .appendTitle(' post') .schema, example: { - plugin: '@data-fair/processing-export-file', + pluginId: '@data-fair/processing-export-file@1', owner: { type: 'organization', id: 'koumoul', @@ -592,317 +594,6 @@ export default (origin: string, options?: { processing?: Processing, plugin?: Pl } } }, - - '/plugins-registry': { - get: { - summary: 'Obtenir la liste des plugins', - description: 'Accéder à la liste des plugins disponibles sur NPM.', - operationId: 'getPluginsRegistry', - tags: ['Plugins'], - parameters: [ - { - name: 'q', - in: 'query', - description: 'Le nom du plugin à rechercher.', - schema: { - type: 'string', - title: 'Nom du plugin à rechercher' - } - }, - { - name: 'showAll', - in: 'query', - description: 'Afficher tous les plugins disponibles (même ceux en version bêta). La requête peut prendre plus de temps.', - schema: { - type: 'boolean', - title: 'Afficher tous les plugins disponibles' - } - } - ], - responses: { - 200: { - description: 'La liste des plugins disponibles', - content: { - 'application/json': { - schema: { - type: 'object', - properties: { - count: { - type: 'integer', - description: 'Le nombre de plugins trouvés.' - }, - results: { - type: 'array', - items: { - type: 'object', - properties: { - name: { - type: 'string', - description: 'Le nom du plugin.' - }, - description: { - type: 'string', - description: 'La description du plugin.' - }, - version: { - type: 'string', - description: 'La version du plugin.' - }, - distTag: { - type: 'string', - description: 'Le tag de distribution du plugin.', - example: 'latest' - } - } - } - } - } - } - } - } - }, - 400: { - description: 'Requête invalide, le paramètre "q" est mal formaté.' - }, - 429: { - description: 'Erreur renvoyée par l\'API NPM, trop de requêtes envoyées.' - }, - 500: { - description: 'Erreur interne (Il se peut que le service NPM soit indisponible).' - } - } - } - }, - '/plugins': { - get: { - summary: 'Obtenir la liste des plugins installés', - description: 'Accéder à la liste des plugins installés.', - operationId: 'getPlugins', - tags: ['Plugins'], - parameters: [ - { - name: 'privateAccess', - in: 'query', - description: 'Filtre par accès', - schema: { - type: 'string', - title: 'Filtre par accès', - example: 'type:id' - } - } - ], - responses: { - 200: { - description: 'La liste des plugins installés', - content: { - 'application/json': { - schema: { - count: { - type: 'integer', - description: 'Le nombre de plugins trouvés.' - }, - facets: { - type: 'object', - properties: { - usages: { - type: 'object', - additionalProperties: { - type: 'integer', - description: 'Le nombre de fois que le plugin est utilisé' - } - } - } - }, - results: { - type: 'array', - items: PluginSchema - } - } - } - } - }, - 400: { - description: 'Le paramètre "privateAccess" est manquant et l\'utilisateur n\'est pas super administrateur.' - }, - 403: { - description: 'Le privateAccess ne correspond pas avec l\'utilisateur authentifié.' - } - } - }, - post: { - summary: 'Installer un plugin', - description: 'Installer/Mettre à jour un plugin, voir même baisser en version un plugin. Cette requête prends beaucoup de temps à s\'executer, c\'est le temps que le plugin s\'installe sur le serveur.', - operationId: 'postPlugin', - tags: ['Plugins'], - requestBody: { - description: 'Le plugin à installer', - required: true, - content: { - 'application/json': { - schema: jsonSchema(PluginSchema) - .pickProperties(['distTag', 'name', 'version', 'description']) - .removeFromRequired(['description']) - .removeId() - .appendTitle(' post') - .schema, - example: { - name: '@data-fair/processing-export-file', - description: 'Export File', - version: '0.6.2', - distTag: 'latest' - } - } - } - }, - responses: { - 200: { - description: 'Le plugin a été installé avec succès.', - content: { - 'application/json': { - schema: PluginSchema - } - } - } - } - } - }, - '/plugins/{id}': { - parameters: [ - { - name: 'id', - in: 'path', - required: true, - description: 'L\'identifiant du plugin.', - schema: { - type: 'string', - title: 'Identifiant du plugin' - } - } - ], - get: { - summary: 'Lire les informations d\'un plugin', - description: 'Accéder aux données d\'un plugin.', - operationId: 'getPlugin', - tags: ['Plugins'], - responses: { - 200: { - description: 'Le plugin trouvé.', - content: { - 'application/json': { - schema: PluginSchema - } - } - }, - 404: { - description: 'Plugin non trouvé.' - } - } - }, - delete: { - summary: 'Supprimer un plugin', - description: 'Supprimer un plugin.', - operationId: 'deletePlugin', - tags: ['Plugins'], - responses: { - 204: { - description: 'Le plugin a été supprimé avec succès.' - } - } - }, - }, - '/plugins/{id}/config': { - parameters: [ - { - name: 'id', - in: 'path', - required: true, - description: 'L\'identifiant du plugin.', - schema: { - type: 'string', - title: 'Identifiant du plugin' - } - } - ], - put: { - summary: 'Mettre à jour la configuration d\'un plugin', - description: 'Mettre à jour la configuration d\'un plugin.', - operationId: 'putPluginConfig', - tags: ['Plugins'], - requestBody: { - description: 'La configuration du plugin', - required: true, - content: { - 'application/json': { - schema: { - type: 'object' - } - } - } - }, - } - }, - '/plugins/{id}/metadata': { - parameters: [ - { - name: 'id', - in: 'path', - required: true, - description: 'L\'identifiant du plugin.', - schema: { - type: 'string', - title: 'Identifiant du plugin' - } - } - ], - put: { - summary: 'Mettre à jour les métadonnées d\'un plugin', - description: 'Mettre à jour les métadonnées d\'un plugin.', - operationId: 'putPluginMetadata', - tags: ['Plugins'], - requestBody: { - description: 'Les métadonnées du plugin', - required: true, - content: { - 'application/json': { - schema: { - type: 'object' - } - } - } - }, - } - }, - '/plugins/{id}/access': { - parameters: [ - { - name: 'id', - in: 'path', - required: true, - description: 'L\'identifiant du plugin.', - schema: { - type: 'string', - title: 'Identifiant du plugin' - } - } - ], - put: { - summary: 'Mettre à jour les accès d\'un plugin', - description: 'Mettre à jour les accès d\'un plugin.', - operationId: 'putPluginAccess', - tags: ['Plugins'], - requestBody: { - description: 'les accès du plugin', - required: true, - content: { - 'application/json': { - schema: { - type: 'object' - } - } - } - }, - } - } } } diff --git a/api/src/plugins-registry/router.ts b/api/src/plugins-registry/router.ts deleted file mode 100644 index 57ac6b80..00000000 --- a/api/src/plugins-registry/router.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { AxiosRequestConfig } from 'axios' -import { Router } from 'express' -import memoize from 'memoizee' - -import { httpError } from '@data-fair/lib-utils/http-errors.js' -import axios from '@data-fair/lib-node/axios.js' -import config from '#config' - -const router = Router() -export default router - -const axiosOpts: AxiosRequestConfig = {} - -if (config.npm?.httpsProxy) { - const proxyUrl = new URL(config.npm?.httpsProxy) - // cf https://axios-http.com/docs/req_config - axiosOpts.proxy = { - protocol: proxyUrl.protocol, - host: proxyUrl.hostname, - port: proxyUrl.port ? Number(proxyUrl.port) : (proxyUrl.protocol === 'https:' ? 443 : 80) - } - if (proxyUrl.username) axiosOpts.proxy.auth = { username: proxyUrl.username, password: proxyUrl.password } -} - -/** - * Search for plugins in the npm registry - * @param q - search query - * @param showAll - if true, return all versions of each plugin - */ -const search = async (q: string | undefined, showAll: boolean) => { - // see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#get-v1search - let res - try { - res = await axios.get('https://registry.npmjs.org/-/v1/search', { - ...axiosOpts, - params: { - size: 250, - text: `keywords:data-fair-processings-plugin ${q || ''}` - } - }) - } catch (error: any) { - if (error.status === 429) throw httpError(429, 'Too many requests to NPM registry') - throw error - } - - const results = [] - for (const o of res.data.objects) { - if (!o.package.keywords || !o.package.keywords.includes('data-fair-processings-plugin')) continue - const plugin = { name: o.package.name, description: o.package.description, version: o.package.version } - - if (showAll) { - const distTags = (await axios.get('https://registry.npmjs.org/-/package/' + o.package.name + '/dist-tags', axiosOpts)).data - for (const distTag in distTags) { - results.push({ ...plugin, version: distTags[distTag], distTag }) - } - } else { - results.push({ ...plugin, distTag: 'latest' }) - } - } - return { - count: results.length, - results - } -} -const memoizedSearch = memoize(search, { - maxAge: 5 * 60 * 1000 // cached for 5 minutes to be polite with npmjs -}) - -router.get('/', async (req, res) => { - if (req.query.q && typeof req.query.q !== 'string') { - res.status(400).send('Invalid query') - return - } - res.send(await memoizedSearch(req.query.q, req.query.showAll === 'true' || false)) -}) diff --git a/api/src/plugins/router.ts b/api/src/plugins/router.ts deleted file mode 100644 index c71deb35..00000000 --- a/api/src/plugins/router.ts +++ /dev/null @@ -1,308 +0,0 @@ -import type { Plugin } from '#types/plugin/index.ts' - -import { exec } from 'child_process' -import { promisify } from 'util' -import { Router } from 'express' -import Ajv from 'ajv' -import ajvFormats from 'ajv-formats' -import fs from 'fs-extra' -import path from 'path' -import semver from 'semver' -import resolvePath from 'resolve-path' -import tmp from 'tmp-promise' -import multer from 'multer' - -import { session, httpError } from '@data-fair/lib-express' -import mongo from '#mongo' -import config from '#config' -import permissions from '../misc/utils/permissions.ts' - -// @ts-ignore -const ajv = ajvFormats(new Ajv({ strict: false })) -const execAsync = promisify(exec) - -const router = Router() -export default router - -const pluginsDir = path.resolve(config.dataDir, 'plugins') -fs.ensureDirSync(pluginsDir) -const tmpDir = config.tmpDir || path.resolve(config.dataDir, 'tmp') -fs.ensureDirSync(tmpDir) - -tmp.setGracefulCleanup() - -/** - * Multer configuration for handling file uploads. - * It stores files directly in the temporary directory and only allows .tgz files. - */ -const upload = multer({ - storage: multer.diskStorage({ - destination: (req, file, cb) => { - cb(null, tmpDir) - }, - filename: (req, file, cb) => { - cb(null, `plugin-${Date.now()}-${file.originalname}`) - } - }), - fileFilter: (req, file, cb) => { - if (path.extname(file.originalname) === '.tgz') cb(null, true) - else cb(httpError(400, 'Only .tgz files are allowed')) - }, - limits: { fileSize: 1 * 1024 * 1024 } // 1MB max -}) - -const pluginMetadataSchema = { - type: 'object', - additionalProperties: false, - properties: { - name: { - type: 'string', - title: 'Nom du plugin', - layout: { - cols: 4 - } - }, - category: { - type: 'string', - title: 'Catégorie', - enum: config.pluginCategories, - layout: { - cols: 4 - } - }, - icon: { - type: 'object', - title: 'Icon', - layout: { - getItems: { - url: 'https://koumoul.com/data-fair/api/v1/datasets/icons-mdi-latest/lines?q={q}&select=name,svg,svgPath&size=25', - itemsResults: 'data.results', - itemTitle: 'item.name', - itemIcon: 'item.svg', - itemKey: 'item.name' - }, - cols: 4 - } - }, - documentation: { - type: 'string', - title: 'Documentation', - description: 'URL de la page du tutoriel du plugin', - format: 'uri', - errorMessage: 'Dois être une URL valide', - }, - description: { - type: 'string', - title: 'Description du plugin' - } - } -} - -// Install a new plugin or update an existing one -router.post('/', upload.single('file'), permissions.isSuperAdmin, async (req, res) => { - const dir = await tmp.dir({ unsafeCleanup: true, tmpdir: tmpDir, prefix: 'plugin-install-' }) - let id: string - let tarballPath: string - let plugin: Partial - - try { - let distTag = 'latest' - if (req.file) { // File upload mode - use the uploaded .tgz file - tarballPath = req.file.path - } else { // NPM mode - validate body and download from npm - const { body } = (await import('#doc/plugin/post-req/index.ts')).returnValid(req) - - // download the plugin package using npm pack - const { stdout } = await execAsync(`npm pack ${body.name}@${body.version}`, { cwd: dir.path }) - tarballPath = path.join(dir.path, stdout.trim()) - if (body.distTag !== 'latest') distTag = body.distTag - } - - // extract the tarball - await execAsync(`tar -xzf "${tarballPath}" -C "${dir.path}"`) - const extractedPath = path.join(dir.path, 'package') - - // install dependencies of the plugin - await execAsync('npm install --omit=dev', { cwd: extractedPath }) - - // generate plugin.json from package.json - const packageJson = await fs.readJson(path.join(extractedPath, 'package.json')) - id = packageJson.name.replace('/', '-') + '-' + semver.major(packageJson.version) - if (distTag !== 'latest') id += '-' + distTag - - plugin = { - id, - name: packageJson.name, - description: packageJson.description, - version: packageJson.version, - distTag - } - - // read plugin schemas - plugin.pluginConfigSchema = await fs.readJson(path.join(extractedPath, 'plugin-config-schema.json')) - plugin.processingConfigSchema = await fs.readJson(path.join(extractedPath, 'processing-config-schema.json')) - - // static metadata for the plugin - await fs.writeFile( - path.join(extractedPath, 'plugin.json'), - JSON.stringify(plugin, null, 2) - ) - - // Create index.js if it doesn't exist and redirects to the main file - if (!await fs.pathExists(path.join(extractedPath, 'index.js'))) { - await fs.writeFile(path.join(extractedPath, 'index.js'), `export * from './${packageJson.main}'`) - } - - // move the extracted plugin to the final destination - await fs.move(extractedPath, path.join(pluginsDir, id), { overwrite: true }) - await dir.cleanup() - if (req.file) await fs.remove(req.file.path) - } catch (error: any) { - await dir.cleanup() - if (req.file) await fs.remove(req.file.path) - throw httpError(400, `Failed to install plugin: ${error.message || error}`) - } - - // set defaults access (don't overwrite if already exists (after an update)) - plugin.access = { public: false, privateAccess: [] } - const accessFilePath = path.join(pluginsDir, plugin.id + '-access.json') - if (!await fs.pathExists(accessFilePath)) await fs.writeJson(accessFilePath, plugin.access) - - // return the existing config if it exists - const pluginConfigPath = path.join(pluginsDir, plugin.id + '-config.json') - if (await fs.pathExists(pluginConfigPath)) plugin.config = await fs.readJson(pluginConfigPath) - - // return the existing metadata if it exists - const pluginMetadataPath = path.join(pluginsDir, plugin.id + '-metadata.json') - if (await fs.pathExists(pluginMetadataPath)) plugin.metadata = await fs.readJson(pluginMetadataPath) - - res.send(plugin) -}) - -// List installed plugins (optional: privateAccess=[type]:[id]) -router.get('/', async (req, res) => { - const sessionState = await session.reqAuthenticated(req) - - const dirs = (await fs.readdir(pluginsDir)).filter(p => !p.endsWith('.json')) - const results: Plugin[] = [] - for (const dir of dirs) { - const pluginInfo: Plugin = await fs.readJson(path.join(pluginsDir, dir, 'plugin.json')) - - let access = { public: false, privateAccess: [] } - const accessFilePath = path.join(pluginsDir, dir + '-access.json') - if (await fs.pathExists(accessFilePath)) { - access = await fs.readJson(path.join(pluginsDir, dir + '-access.json')) - } - - if (sessionState.user.adminMode) { - const pluginConfigPath = path.join(pluginsDir, dir + '-config.json') - if (await fs.pathExists(pluginConfigPath)) pluginInfo.config = await fs.readJson(pluginConfigPath) - pluginInfo.access = access - } else if (req.query.privateAccess && typeof req.query.privateAccess === 'string') { - const [type, id] = req.query.privateAccess.split(':') - if (type !== sessionState.account.type || id !== sessionState.account.id) { - throw httpError(403, 'privateAccess does not match current account') - } - if (!access.public && !access.privateAccess.find((p: any) => p.type === type && p.id === id)) { - continue // pass to next plugin - } - } else { - throw httpError(400, 'privateAccess filter is required') - } - - const pluginMetadataPath = path.join(pluginsDir, dir + '-metadata.json') - const version = pluginInfo.distTag === 'latest' ? pluginInfo.version : `${pluginInfo.distTag} - ${pluginInfo.version}` - pluginInfo.metadata = { - name: pluginInfo.name.replace('@data-fair/processing-', '') + ' (' + version + ')', - description: pluginInfo.description, - ...(await fs.pathExists(pluginMetadataPath) ? await fs.readJson(pluginMetadataPath) : {}) - } - pluginInfo.pluginMetadataSchema = pluginMetadataSchema - - results.push(pluginInfo) - } - - const aggregationResult = ( - await mongo.processings - .aggregate([{ $group: { _id: '$plugin', count: { $sum: 1 } } }]) - .toArray() - ).reduce((acc:any, { _id, count }: any) => { - acc[_id] = count - return acc - }, {}) - - res.send({ - count: results.length, - results, - facets: { usages: aggregationResult || {} } - }) -}) - -// Return PluginData (if connected) -router.get('/:id', async (req, res) => { - await session.reqAuthenticated(req) - try { - const pluginInfo: Plugin = await fs.readJson(resolvePath(pluginsDir, path.join(req.params.id, 'plugin.json'))) - const pluginMetadataPath = path.join(pluginsDir, req.params.id + '-metadata.json') - const version = pluginInfo.distTag === 'latest' ? pluginInfo.version : `${pluginInfo.distTag} - ${pluginInfo.version}` - pluginInfo.metadata = { - name: pluginInfo.name.replace('@data-fair/processing-', '') + ' (' + version + ')', - description: pluginInfo.description, - ...(await fs.pathExists(pluginMetadataPath) ? await fs.readJson(pluginMetadataPath) : {}) - } - res.send(pluginInfo) - } catch (e: any) { - if (e.code === 'ENOENT') res.status(404).send('Plugin not found') - else throw e - } -}) - -router.delete('/:id', permissions.isSuperAdmin, async (req, res) => { - const id = req.params.id as string - if (!id) throw httpError(400, 'Plugin ID is required') - if (!fs.existsSync(path.join(pluginsDir, id))) throw httpError(404, 'Plugin not found') - - await fs.remove(path.join(pluginsDir, id)) - await fs.remove(path.join(pluginsDir, id + '-config.json')) - await fs.remove(path.join(pluginsDir, id + '-access.json')) - await fs.remove(path.join(pluginsDir, id + '-metadata.json')) - res.status(204).send() -}) - -router.put('/:id/config', permissions.isSuperAdmin, async (req, res) => { - const id = req.params.id as string - const pluginPath = path.join(pluginsDir, id, 'plugin.json') - if (!await fs.pathExists(pluginPath)) { - throw httpError(404, 'Plugin not found') - } - - const { pluginConfigSchema } = await fs.readJson(pluginPath) - const validate = ajv.compile(pluginConfigSchema) - const valid = validate(req.body) - if (!valid) return res.status(400).send(validate.errors) - await fs.writeJson(path.join(pluginsDir, id + '-config.json'), req.body) - res.send(req.body) -}) - -router.put('/:id/metadata', permissions.isSuperAdmin, async (req, res) => { - const id = req.params.id as string - if (!await fs.pathExists(path.join(pluginsDir, id, 'plugin.json'))) { - throw httpError(404, 'Plugin not found') - } - - const validate = ajv.compile(pluginMetadataSchema) - const valid = validate(req.body) - if (!valid) return res.status(400).send(validate.errors) - await fs.writeJson(path.join(pluginsDir, id + '-metadata.json'), req.body) - res.send(req.body) -}) - -router.put('/:id/access', permissions.isSuperAdmin, async (req, res) => { - const id = req.params.id as string - if (!await fs.pathExists(path.join(pluginsDir, id, 'plugin.json'))) { - throw httpError(404, 'Plugin not found') - } - - await fs.writeJson(path.join(pluginsDir, id + '-access.json'), req.body) - res.send(req.body) -}) diff --git a/api/src/processings/router.ts b/api/src/processings/router.ts index 1a4daf6a..aec25651 100644 --- a/api/src/processings/router.ts +++ b/api/src/processings/router.ts @@ -8,17 +8,18 @@ import cryptoRandomString from 'crypto-random-string' import { Router } from 'express' import fs from 'fs-extra' import path from 'path' -import resolvePath from 'resolve-path' import { nanoid } from 'nanoid' import eventsQueue from '@data-fair/lib-node/events-queue.js' import { reqOrigin, session } from '@data-fair/lib-express/index.js' +import { ensureArtefact } from '@data-fair/lib-node-registry' import { httpError } from '@data-fair/lib-utils/http-errors.js' import { createNext } from '@data-fair/processings-shared/runs.ts' +import { parsePluginId } from '@data-fair/processings-shared/plugin-id.ts' import { applyProcessing, deleteProcessing } from '../runs/service.ts' import { cipher, decipher } from '@data-fair/processings-shared/cipher.ts' import mongo from '#mongo' -import config from '#config' +import config, { registryCacheDir } from '#config' import locks from '#locks' import { resolvedSchema as processingSchema } from '#types/processing/index.ts' import getApiDoc from '../misc/utils/api-docs.ts' @@ -30,7 +31,33 @@ export default router // @ts-ignore const ajv = ajvFormats(new Ajv({ strict: false })) -const pluginsDir = path.join(config.dataDir, 'plugins') + +/** + * Ensure the plugin tarball is in the API's local cache (downloads on miss), + * then read its package.json. Registry enforces that `processing.owner` has + * access to the artefact; a 403 here surfaces directly to the caller. + */ +async function ensurePluginAndReadSchema (processing: Pick) { + const { name, major } = parsePluginId(processing.pluginId) + const ensured = await ensureArtefact({ + registryUrl: config.privateRegistryUrl, + secretKey: config.secretKeys.registry, + artefactId: name, + version: major, + cacheDir: registryCacheDir, + architecture: process.arch, + account: { + type: processing.owner.type, + id: processing.owner.id, + ...(processing.owner.department ? { department: processing.owner.department } : {}) + } + }) + const pkg = await fs.readJson(path.join(ensured.path, 'package.json')) + return { + ensured, + processingConfigSchema: pkg.registry?.processingConfigSchema as Record | undefined + } +} const sensitiveParts = ['permissions', 'webhookKey', 'config'] @@ -62,26 +89,32 @@ const sendProcessingEvent = ( } /** - * Check that a processing object is valid - * Check if the plugin exists - * Check if the config is valid (only if the processing is activated) - * Encrypt secrets if present + * Check that a processing object is valid: + * - run schema validation + * - require a config when active + * - validate the config against the plugin's processingConfigSchema (read + * from the cached package.json#registry block, downloaded on cache miss) + * + * Errors from registry (404 unknown artefact, 403 owner has no access) bubble + * up as the corresponding HTTP errors and replace the explicit access check + * that used to live in the create/update endpoints. */ async function validateFullProcessing (processing: any): Promise { (await import('#types/processing/index.ts')).returnValid(processing) if (processing.active && !processing.config) throw httpError(400, 'Config is required for an active processing') - if (!await fs.pathExists(path.join(pluginsDir, processing.plugin))) throw httpError(400, 'Plugin not found') if (!processing.config) return processing // no config to validate - const pluginInfo = await fs.readJson(path.join(pluginsDir, processing.plugin, 'plugin.json')) - const configValidate = ajv.compile(pluginInfo.processingConfigSchema) + const { processingConfigSchema } = await ensurePluginAndReadSchema(processing) + if (!processingConfigSchema) throw httpError(400, 'plugin has no processingConfigSchema') + const configValidate = ajv.compile(processingConfigSchema) const configValid = configValidate(processing.config) if (!configValid) throw httpError(400, JSON.stringify(configValidate.errors)) return processing } const prepareProcessing = async (processing: Processing) => { - // Get the plugin file and execute the prepare function if it exists - const plugin = await import(path.resolve(process.cwd(), pluginsDir, processing.plugin, 'index.js') + `?imported=${Date.now()}`) + // Get the plugin file and execute the prepare function if it exists. + const { ensured } = await ensurePluginAndReadSchema(processing) + const plugin = await import(path.join(ensured.path, 'index.js') + `?imported=${Date.now()}`) if (!(plugin.prepare && typeof plugin.prepare === 'function')) return // Decipher the actuals secrets if they are present @@ -144,10 +177,10 @@ router.get('', async (req, res) => { ].filter(Boolean) }) } - // Filter by plugins + // Filter by plugins (matches the denormalized pluginId, e.g. `@scope/foo@1`) const plugins = params.plugins ? params.plugins.split(',') : [] if (plugins.length > 0) { - queryWithFilters.plugin = { $in: plugins } + queryWithFilters.pluginId = { $in: plugins } } // Get the processings @@ -187,7 +220,7 @@ router.get('', async (req, res) => { plugins: [ { $group: { - _id: '$plugin', + _id: '$pluginId', count: { $sum: 1 } } } @@ -297,17 +330,9 @@ router.post('', async (req, res) => { date: new Date().toISOString() } - const access = await fs.pathExists(resolvePath(pluginsDir, body.plugin + '-access.json')) ? await fs.readJson(resolvePath(pluginsDir, body.plugin + '-access.json')) : { public: false, privateAccess: [] } - if (sessionState.user.adminMode) { - // ok for super admins - } else if (access && access.public) { - // ok, this plugin is public - } else if (access && access.privateAccess && access.privateAccess.find((p: any) => p.type === body.owner.type && p.id === body.owner.id)) { - // ok, private access is granted - } else { - return res.status(403).send('Access denied to this plugin') - } - + // Plugin access check is delegated to registry: validateFullProcessing + // calls ensureArtefact with owner=body.owner, and registry returns 403 if + // the artefact is private and the owner has no privateAccess entry. const processing = await validateFullProcessing(body) Object.assign(processing, await prepareProcessing(processing)) await mongo.processings.insertOne(processing) @@ -383,6 +408,22 @@ router.get('/:id', async (req, res) => { res.status(200).json(cleanProcessing(processing, sessionState)) }) +// Get the plugin's processingConfigSchema for this processing's pinned major. +// Registry only stores tarballs and ignores their inner shape; the schema +// lives in the package.json under `registry.processingConfigSchema` and is +// read on demand from the API's local cache. +router.get('/:id/plugin-config-schema', async (req, res, next) => { + try { + const sessionState = await session.reqAuthenticated(req) + const processing = await mongo.processings.findOne({ _id: req.params.id }) + if (!processing) return res.status(404).send() + if (!['admin', 'exec', 'read'].includes(permissions.getUserResourceProfile(processing.owner, processing.permissions, sessionState) ?? '')) return res.status(403).send() + const { processingConfigSchema } = await ensurePluginAndReadSchema(processing) + if (!processingConfigSchema) return res.status(404).send() + res.status(200).json(processingConfigSchema) + } catch (err) { next(err) } +}) + // Delete a processing router.delete('/:id', async (req, res) => { const sessionState = await session.reqAuthenticated(req) @@ -432,11 +473,17 @@ router.post('/:id/_trigger', async (req, res) => { }) // Get the API documentation of a processing -router.get('/:id/api-docs.json', permissions.isSuperAdmin, async (req, res) => { - const processing = await mongo.processings.findOne({ _id: req.params.id }) - if (!processing) return res.status(404).send() - const pluginPath = path.join(pluginsDir, processing.plugin, 'plugin.json') - if (!await fs.pathExists(pluginPath)) return res.status(404).send('Plugin not found') - const plugin = await fs.readJson(pluginPath) - res.json(getApiDoc(reqOrigin(req), { processing, plugin })) +router.get('/:id/api-docs.json', permissions.isSuperAdmin, async (req, res, next) => { + try { + const processing = await mongo.processings.findOne({ _id: req.params.id }) + if (!processing) return res.status(404).send() + const { ensured, processingConfigSchema } = await ensurePluginAndReadSchema(processing) + const pluginPkg = await fs.readJson(path.join(ensured.path, 'package.json')) + const plugin = { + name: pluginPkg.name, + version: pluginPkg.version, + processingConfigSchema + } + res.json(getApiDoc(reqOrigin(req), { processing, plugin })) + } catch (err) { next(err) } }) diff --git a/api/src/runs/service.ts b/api/src/runs/service.ts index 1d9d2633..6c5e22a5 100644 --- a/api/src/runs/service.ts +++ b/api/src/runs/service.ts @@ -8,7 +8,7 @@ import path from 'path' import resolvePath from 'resolve-path' import locks from '#locks' -const processingsDir = path.resolve(config.dataDir, 'processings') +const processingsDir = path.resolve(config.dataDir ?? config.tmpDir, 'processings') /** * Stop all pending runs for a processing if it is deactivated diff --git a/api/src/server.ts b/api/src/server.ts index f2ab198f..f0401db6 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -26,7 +26,7 @@ server.keepAliveTimeout = (60 * 1000) + 1000 server.headersTimeout = (60 * 1000) + 2000 export const start = async () => { - if (!existsSync(config.dataDir) && process.env.NODE_ENV === 'production') { + if (config.dataDir && !existsSync(config.dataDir) && process.env.NODE_ENV === 'production') { throw new Error(`Data directory ${resolvePath(config.dataDir)} was not mounted`) } if (config.observer.active) await startObserver(config.observer.port) diff --git a/api/types/index.ts b/api/types/index.ts index 6eb18dc1..d7519002 100644 --- a/api/types/index.ts +++ b/api/types/index.ts @@ -2,7 +2,6 @@ import type { Account } from '@data-fair/lib-express' export type { Limit } from './limit/index.js' export type { Permission } from './permission/index.js' -export type { Plugin } from './plugin/index.js' export type { Processing } from './processing/index.js' export type { Run } from './run/index.js' export type { Scheduling } from './scheduling/index.js' diff --git a/api/types/plugin/index.ts b/api/types/plugin/index.ts deleted file mode 100644 index f5d615bd..00000000 --- a/api/types/plugin/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './.type/index.js' diff --git a/api/types/plugin/schema.js b/api/types/plugin/schema.js deleted file mode 100644 index 3037c28d..00000000 --- a/api/types/plugin/schema.js +++ /dev/null @@ -1,103 +0,0 @@ -export default { - $id: 'https://github.com/data-fair/processings/plugin', - 'x-exports': [ - 'types', - 'validate', - 'resolvedSchema' - ], - title: 'plugin', - type: 'object', - additionalProperties: false, - required: [ - 'name', - 'description', - 'version', - 'distTag', - 'id', - 'metadata', - 'pluginConfigSchema', - 'pluginMetadataSchema', - 'processingConfigSchema' - ], - properties: { - name: { - type: 'string', - }, - description: { - type: 'string' - }, - version: { - type: 'string' - }, - distTag: { - type: 'string' - }, - id: { - type: 'string' - }, - pluginConfigSchema: { - type: 'object', - description: 'Schema de configuration du plugin.', - }, - pluginMetadataSchema: { - type: 'object', - description: 'Schema de configuration des metadata.' - }, - processingConfigSchema: { - type: 'object', - description: 'Schema de configuration du traitement.' - }, - config: { - type: 'object', - description: 'La configuration du plugin respectant le schema de configuration du plugin.', - }, - access: { - type: 'object', - additionalProperties: false, - properties: { - public: { - type: 'boolean' - }, - privateAccess: { - type: 'array', - items: { - type: 'object', - additionalProperties: false, - required: ['type', 'id'], - properties: { - type: { - type: 'string' - }, - id: { - type: 'string' - } - } - } - } - } - }, - metadata: { - type: 'object', - description: 'Les metadata du plugin respectant le schema de configuration des metadata.', - additionalProperties: false, - required: ['name', 'description', 'category', 'icon'], - properties: { - name: { - type: 'string' - }, - description: { - type: 'string' - }, - category: { - type: 'string' - }, - icon: { - type: 'string' - }, - documentation: { - type: 'string' - } - } - } - } -} diff --git a/api/types/processing/schema.js b/api/types/processing/schema.js index b65c2ac3..58b86df9 100644 --- a/api/types/processing/schema.js +++ b/api/types/processing/schema.js @@ -14,7 +14,7 @@ export default { required: [ '_id', 'owner', - 'plugin', + 'pluginId', 'scheduling', 'title' ], @@ -84,9 +84,10 @@ export default { $ref: 'https://github.com/data-fair/lib/account', readOnly: true }, - plugin: { + pluginId: { type: 'string', - readOnly: true + readOnly: true, + description: 'Registry artefact id "{name}@{majorVersion}" (e.g. @data-fair/processing-hello-world@1). Patch upgrades inside a major are implicit — registry resolves the latest available version at run time.' }, scheduling: { type: 'array', diff --git a/dev/init-env.sh b/dev/init-env.sh index 2cb380d1..da2a1946 100755 --- a/dev/init-env.sh +++ b/dev/init-env.sh @@ -26,4 +26,5 @@ SD_PORT=$((RANDOM_NB + 30)) EVENTS_PORT=$((RANDOM_NB + 31)) OAV_PORT=$((RANDOM_NB + 32)) DF_PORT=$((RANDOM_NB + 33)) +REGISTRY_PORT=$((RANDOM_NB + 34)) EOF diff --git a/dev/resources/nginx.conf.template b/dev/resources/nginx.conf.template index e29c20ca..5d2c256f 100644 --- a/dev/resources/nginx.conf.template +++ b/dev/resources/nginx.conf.template @@ -66,4 +66,8 @@ server { location /events/ { proxy_pass http://localhost:${EVENTS_PORT}/; } + + location /registry/ { + proxy_pass http://localhost:${REGISTRY_PORT}/registry/; + } } diff --git a/docker-compose.yml b/docker-compose.yml index e3dfa046..b27e8cfb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: - DF_PORT=${DF_PORT} - EVENTS_PORT=${EVENTS_PORT} - OAV_PORT=${OAV_PORT} + - REGISTRY_PORT=${REGISTRY_PORT} volumes: - ./dev/resources/nginx.conf.template:/etc/nginx/templates/default.conf.template:ro @@ -106,6 +107,29 @@ services: - USE_SIMPLE_DIRECTORY=true - ALLOWED_URLS={"processings":"http://${DEV_HOST}:${NGINX_PORT1}/processings/api/v1/api-docs.json","processingsId":"http://${DEV_HOST}:${NGINX_PORT1}/processings/api/v1/processings/{id}/api-docs.json","processingsAdmin":"http://${DEV_HOST}:${NGINX_PORT1}/processings/api/v1/admin/api-docs.json"} + # registry-side dev container — same domain as processings under /registry + # so the UI can hit it directly with the SimpleDirectory session cookie. + registry: + profiles: + - dev + image: ghcr.io/data-fair/registry:feat-processings-integration + network_mode: host + depends_on: + - mongo + - simple-directory + environment: + - PORT=${REGISTRY_PORT} + - MONGO_URL=mongodb://localhost:${MONGO_PORT}/data-fair-registry + - PRIVATE_DIRECTORY_URL=http://localhost:${SD_PORT} + - PUBLIC_URL=http://${DEV_HOST}:${NGINX_PORT1}/registry + - SECRET_INTERNAL_SERVICES=secret-registry-internal + - CIPHER_PASSWORD=1234567890123456789012345678901234567890 + - API_KEYS_SALT=1234567890123456789012345678901234567890 + - OBSERVER_ACTIVE=false + - STORAGE_TYPE=fs + volumes: + - registry-data:/app/data + ##### # db and search engine ##### @@ -183,3 +207,4 @@ volumes: elasticsearch-data: mongo-data: processings-data: + registry-data: diff --git a/docs/architecture/v6-registry-integration.md b/docs/architecture/v6-registry-integration.md new file mode 100644 index 00000000..38b3702b --- /dev/null +++ b/docs/architecture/v6-registry-integration.md @@ -0,0 +1,333 @@ +# Processings v6.0 — registry integration + +## Context + +`@data-fair/processings` today stores plugin code on a persistent volume at `dataDir/plugins/{plugin-id}` and tracks per-plugin state in three sibling JSON files (`-access.json`, `-config.json`, `-metadata.json`). Installation goes through a superadmin-only `POST /api/v1/plugins` that runs `npm pack` + `npm install --omit=dev`. The API and worker both `import()` directly from the volume. + +We want to move plugin code, schemas, metadata and access fully into `@data-fair/registry` (sibling repo at `/home/alban/data-fair/registry`), so processings becomes stateless w.r.t. plugin content. The worker and API will use registry's `lib-node` to download plugin tarballs into a per-container ephemeral cache (k8s `emptyDir`) on demand. + +This is a major version (v6.0) and a transition release: the legacy plugins volume is still optionally mounted, but only as a *read-only* source for two things: + +1. The first-boot migration — it reads each plugin dir, re-tars it (with `node_modules`) and uploads to registry, plus pushes metadata and access-grants. +2. Legacy plugin-config (the editable global per-instance plugin config) — the worker keeps reading `{id}-config.json` from the volume in v6.0 only. The concept is deprecated; new plugin versions must not depend on it. A future major (v7.0) drops both the volume mount and the read-only support. + +Registry is the single source of truth for plugin code, schemas, public/private flag, access-grants, metadata. Processings keeps *no* mongo collection mirroring plugin state. + +## Key decisions + +- Registry holds plugin code + schemas + metadata + access. Processings no longer has an "installed plugins" set. +- Tarballs in registry are pre-installed (include prod `node_modules`) and tagged with the CPU architecture they were built for. lib-node extracts directly; no runtime npm install. +- Architecture-aware retrieval is incomplete in registry today — must be finished. Resolver and tarball download accept an `architecture` query; lib-node detects `process.arch` and sends it; a tarball with no architecture acts as a noarch fallback. +- Worker auth: `x-secret-key` plus `x-account` JSON header. Registry validates the secret AND enforces grants for the supplied account. Without `x-account`, today's full-bypass behaviour is preserved. +- The `POST/GET/PUT/DELETE /api/v1/plugins` endpoints, the on-disk plugin dir at runtime, and the UI plugin admin page are dropped entirely. The UI's "create processing" plugin picker calls registry directly. +- API still calls each plugin's `prepare(context)` hook on save — it uses lib-node to ensure the plugin into its own emptyDir cache, separate from the worker's cache. +- distTag-suffixed legacy plugins (e.g. `foo-1-beta`): skip with warning at migration time. Registry has no distTag concept and mangling artefact names is rejected as ugly; operator handles them manually. +- UI → registry: direct browser call. Registry is deployed on the same domain at the `/registry` path, so no CORS handling is needed; the SimpleDirectory session cookie is already valid for the registry origin. +- Plugin schemas (`processingConfigSchema`) are read by processings from the cached plugin tarball's `package.json#registry`, not from registry artefact metadata. Registry remains a generic artefact store. Registry still extracts the field today for its own UI; that extraction stays for now and may be cleaned up in a separate later effort. +- Documentation: add to registry artefact schema. Icon: convert mdi name to SVG and upload to artefact thumbnail. +- URL config: only `privateRegistryUrl` (internal, used by API/worker server-to-server) is configurable. The UI assumes registry is mounted at `/registry` on the same domain as processings. +- Orphan processings (legacy `processing.plugin` string with no matching plugin dir on volume) → migration fails fast. We don't want a half-migrated database. +- Plugin picker UI in v6.0: flat list with search, no sub-categories. Revisit later. + +## Plan + +### A. Registry-side changes (`/home/alban/data-fair/registry`) + +#### A.1 Architecture-aware version resolution +- `api/src/artefacts/service-pure.ts` — extend `resolveVersionQuery(artefactId, versionParam, architecture?)` to return `{ primaryFilter, fallbackFilter?, sort }`. When `architecture` is set, primary filter adds `{ architecture }`; fallback filter adds `{ architecture: { $exists: false } }`. +- `api/src/artefacts/router.ts` — in `GET /artefacts/:id/versions/:version` and `GET /artefacts/:id/versions/:version/tarball`, read `req.query.architecture`, run `findOne(primary)` then `findOne(fallback)` if missing; return 404 if neither matches. +- Note: when patch 5 exists only as `arm64` and patch 4 as both archs, an `x64` worker resolving `1.2` gets patch 4. That's "latest patch available on my arch", not "latest patch overall". Document explicitly. + +#### A.2 lib-node (`lib-node/index.ts`) +- Add `architecture?: string` (default `process.arch`) and `account?: { type: 'user'|'organization', id: string, department?: string }` to `EnsureArtefactOpts`. +- Pass `architecture` as query param on both the resolve call and the tarball call. +- When `account` is set, send `x-account: JSON.stringify(account)` alongside `x-secret-key`. +- Cache key includes architecture: `extractDir = join(artefactDir, ${resolvedVersion}${arch ? '_' + arch : ''})`. `.current-version.json` becomes `{ version, architecture? }` (additive, backwards-compatible). +- Update `lib-node/index.d.ts` and JSDoc to describe noarch fallback semantics. + +#### A.3 Internal-secret + account context (`api/src/auth.ts` and `api/src/artefacts/router.ts`) +- New helper `tryInternalSecretWithAccount(req)` — returns `null`, `{ account: null }` (full bypass), or `{ account }` (validated). +- `x-account` parses to `{ type, id, department? }`; invalid JSON/type → 400. +- Replace internal-secret usage in `GET /artefacts`, `GET /artefacts/:id`, the resolver, the tarball endpoint, and `GET /artefacts/:id/download`: + - `account === null` → preserve today's bypass (filter: `{}`). + - `account` present → call `artefactAccessFilterForAccount(account, { requireGrant: false })` and apply that filter. +- `api/src/access.ts` — add `requireGrant` option to `artefactAccessFilterForAccount`. Internal-with-account flows pass `false` (an account can be enforced without holding a global grant, as long as the artefact is public or has explicit privateAccess for that account). Read-key flow keeps `requireGrant: true`. +- `PATCH /artefacts/:id` and `POST /access-grants` — currently require `session.reqAdminMode(req)`. Extend both to also accept `x-secret-key` (same temporary internal-secret pattern as the upload endpoint). The migration in section D needs this. + +#### A.4 Documentation field on artefact schema +- `api/types/artefact/schema.js` — add optional `documentation: { type: 'string', format: 'uri' }`. +- Surface it in `GET /artefacts/:id` responses (no extra work — present-in-doc auto-passes through). +- Optional follow-up: extract `documentation` from `package.json#registry.documentation` during upload (mirrors how `processingConfigSchema` is extracted today). + +#### A.5 Tests +- `tests/features/artefacts/*.unit.spec.ts` — new cases for `resolveVersionQuery` arch primary/fallback. +- `tests/features/artefacts/*.api.spec.ts`: + - resolves to arch-specific tarball when one exists + - falls back to noarch tarball when no arch match exists + - 404 if no arch match and no noarch + - download with `x-account` enforces that account's grants + - download without `x-account` preserves bypass + - `PATCH /artefacts/:id` works with `x-secret-key` + +### B. Processings — runtime (`/home/alban/data-fair/processings_feat-registry`) + +#### B.1 Config keys +- `api/config/type/schema.json` and `worker/config/type/schema.json` — top-level `privateRegistryUrl` (required), `secretKeys.registry` (alongside other internal secrets). `dataDir` is optional (defaults to null) and, when set, implies the legacy plugins volume is mounted (no separate flag). +- `api/config/default.mjs`, `worker/config/default.mjs` — sensible defaults. `tmpDir` defaults to `${dataDir}/tmp` if `dataDir` is set, else `${os.tmpdir()}/data-fair-processings`. The registry tarball cache is always derived as `${tmpDir}/registry-cache` (mount `tmpDir` as emptyDir in k8s if you want the cache ephemeral). +- `api/config/custom-environment-variables.mjs`, `worker/config/custom-environment-variables.mjs` — env var names: `PRIVATE_REGISTRY_URL`, `SECRET_REGISTRY`, `DATA_DIR`, `TMP_DIR`. +- `api/src/config.ts` — UI hits `/registry` directly (same-domain assumption), so `uiConfig` no longer carries a registry URL. + +#### B.2 Drop plugin admin +Files to delete entirely: +- `api/src/plugins/router.ts` +- `api/src/plugins-registry/router.ts` +- `api/types/plugin/` +- `api/doc/plugin/` (if present) +- `ui/src/pages/admin/plugins.vue` (and route entry) +- `tests/features/plugins/install.api.spec.ts` +- `tests/features/plugins/registry.api.spec.ts` + +Edits: +- `api/src/app.ts` — remove the two `app.use('/api/v1/plugins-registry', ...)` and `app.use('/api/v1/plugins', ...)` mounts and their imports. +- `ui/src/App.vue` (or wherever the admin nav is built) — drop the `/admin/plugins` link. +- Search `grep -rn 'api/v1/plugins' api/ ui/` and remove every remaining reference. + +#### B.3 Worker — replace dynamic import (`worker/src/task/task.ts` ~ lines 88–134) +Replace the on-disk plugin location block with: + +```ts +import { ensureArtefact } from '@data-fair/registry/lib-node' + +const ensured = await ensureArtefact({ + registryUrl: config.privateRegistryUrl, + secretKey: config.secretKeys.registry, + artefactId: `${processing.plugin.name}@${semver.major(processing.plugin.version)}`, + version: processing.plugin.version, + cacheDir: registryCacheDir, + architecture: process.arch, + account: processing.owner +}) + +let pluginConfig: Record = {} +if (config.dataDir) { + const legacyConfigPath = path.join(config.dataDir, 'plugins', legacyIdFor(processing.plugin) + '-config.json') + if (await fs.pathExists(legacyConfigPath)) pluginConfig = await fs.readJson(legacyConfigPath) +} + +const pluginModule = await import(path.join(ensured.path, 'index.js')) +``` + +`legacyIdFor({name, version})` reproduces the v5 id (`name.replace('/', '-') + '-' + semver.major(version)`). It lives in a small util module used by the migration scripts and the legacy plugin-config lookup; deleted in v7.0. + +The rest of `task.ts` (cwd, processingConfig, secrets, axios, ws, log) stays unchanged. + +#### B.4 API — validation + prepare (`api/src/processings/router.ts`) + +Both validation and `prepare` share one path: `ensureArtefact` into `registryCacheDir`, then read `${ensured.path}/package.json#registry.processingConfigSchema` for ajv. Calls to registry use `x-secret-key` + `x-account: processing.owner`, so a 403 from `ensureArtefact` becomes a 403 from the save endpoint — replaces the explicit access check at L300–309. + +A small helper: + +```ts +async function ensurePluginAndReadSchema(processing) { + const ensured = await ensureArtefact({ + registryUrl: config.privateRegistryUrl, + secretKey: config.secretKeys.registry, + artefactId: `${processing.plugin.name}@${semver.major(processing.plugin.version)}`, + version: processing.plugin.version, + cacheDir: registryCacheDir, + architecture: process.arch, + account: processing.owner + }) + const pkg = await fs.readJson(path.join(ensured.path, 'package.json')) + return { ensured, processingConfigSchema: pkg.registry?.processingConfigSchema } +} +``` + +- `validateFullProcessing` calls `ensurePluginAndReadSchema`, runs ajv on `processing.config`. The cache means the second call within the same save (for `prepare`) doesn't redownload. +- `prepareProcessing` reuses the `ensured` reference if already obtained, otherwise calls the helper. Then `import(path.join(ensured.path, 'index.js') + '?imported=' + Date.now())` (cache-busting because prepare may be invoked multiple times in the same process). Calls `plugin.prepare(context)` if exported. +- `GET /api/v1/processings/{id}/api-docs.json` (router.ts ~L435) calls `ensurePluginAndReadSchema` and inlines the schema into the OpenAPI definition. + +#### B.5 Plugin-config legacy read +Single read site, in worker `task.ts` (B.3). API never reads plugin-config. When `dataDir` is unset, `pluginConfig = {}`. Worker logs a one-line deprecation warning when injecting a non-empty `pluginConfig`, so we can monitor stragglers. + +#### B.6 `processing.plugin` schema change +- `api/types/processing/schema.js` — change the `plugin` property: + ```js + plugin: { + type: 'object', + additionalProperties: false, + required: ['name', 'version'], + readOnly: true, + properties: { + name: { type: 'string' }, + version: { type: 'string', description: 'Semver pin like "1", "1.2", or exact "1.2.3"' } + } + } + ``` +- Aggregations in `router.ts` that group by `$plugin` (~L189, ~L206) — switch to a denormalized helper `pluginId` field updated on every save (`${plugin.name}@${semver.major(plugin.version)}`), or use a `$concat` in the aggregation. +- Mongo migration: see D.4. + +### C. UI changes + +#### C.1 Plugin picker +`ui/src/pages/processings/new.vue` (or wherever the picker lives — `grep -rn 'plugins?privateAccess' ui/src`): + +```ts +const installedPluginsFetch = useFetch<{ results: Artefact[], count: number }>( + `${$uiConfig.registryUrl}/api/v1/artefacts?category=processing&size=100` +) +``` + +Field mapping when displaying / when constructing `processing.plugin`: +- `plugin.metadata.name` ← `artefact.title?.fr ?? artefact.title?.en ?? artefact.name` +- `plugin.metadata.icon` → fetch from `${$uiConfig.registryUrl}/api/v1/artefacts/${artefact._id}/thumbnail/${artefact.thumbnail.id}` +- `processing.plugin` → `{ name: artefact.name, version: String(artefact.majorVersion) }` + +For v6.0, list plugins flat with a search box. No sub-categories. + +Same-domain deployment: registry sits on the same host at `/registry`, so the browser call to `/registry/api/v1/artefacts?...` is same-origin. No CORS configuration is required and the existing SimpleDirectory session cookie is naturally sent. The dev environment must mirror this — see E.1. + +#### C.2 Drop admin pages and nav +- Delete `ui/src/pages/admin/plugins.vue` and matching route in `ui/src/router/`. +- Drop the `/admin/plugins` nav entry. +- Search `grep -rn 'plugins' ui/src/components/admin/ ui/src/App.vue ui/src/i18n/` and clean up. + +### D. First-boot migration (`upgrade/6.0.0/`) + +Lives in the existing upgrade-scripts framework — `worker/src/worker.ts` already calls `await upgradeScripts(mongo.db, locks, config.upgradeRoot)` at startup, which provides the mongo lock against multi-pod races. Three scripts, run in order: + +#### D.1 `01-publish-plugins-to-registry.ts` +- If `dataDir/plugins` doesn't exist → log "no legacy volume, skipping" and return. +- For each plugin dir (skip files matching `*.json`, skip distTag dirs ending `-{distTag}` where `distTag !== 'latest'` — log a warning and continue): + - Read `dataDir/plugins/{id}/plugin.json` → `{ name, version }`. + - Probe registry: `GET /api/v1/artefacts/${encodeURIComponent(name + '@' + semver.major(version))}/versions/${version}?architecture=${process.arch}`. If 200 → already published, skip. + - Re-tar the directory (with `node_modules`) using `tar.create({ gzip: true, cwd: dataDir/plugins, prefix: 'package/' }, [id])`. + - Multipart POST to `${registry.privateUrl}/api/v1/artefacts/${encodeURIComponent(name)}/versions` with `architecture=${process.arch}` and `x-secret-key`. + - On 409 (artefact origin set / mirrored from elsewhere): fail fast with an operator-actionable error. + +#### D.2 `02-push-metadata-and-access.ts` +For each plugin dir: +- Read `{id}-metadata.json` (if present) and `{id}-access.json` (if present, default `{public: false, privateAccess: []}`). +- PATCH `${registry.privateUrl}/api/v1/artefacts/${encodeURIComponent(name + '@' + semver.major(version))}` with `x-secret-key`: + - `public: !!access.public` + - `privateAccess: access.privateAccess` + - `title: { fr: metadata.name }` if present + - `description: { fr: metadata.description }` if present (note: registry localised description; mapping to `fr` matches today's UI) + - `documentation: metadata.documentation` if present (registry artefact schema is being extended in A.4) +- If `metadata.icon` is an mdi name string: render to SVG (use `@mdi/js` to fetch the path, wrap in an `` template) and POST it to the artefact's thumbnail endpoint. If the icon name is invalid, skip with warning. +- For each `access.privateAccess` entry: `POST /api/v1/access-grants` with `{ account: acc }`. Accept 201 or 409 (already exists, idempotent). + +#### D.3 `03-rewrite-processing-plugin.ts` +- Find `processings` where `plugin` is type string. +- For each: parse `{id} = plugin`, find the matching legacy plugin dir, read its `plugin.json` to recover `name` and `version`, then `db.collection('processings').updateOne({_id}, { $set: { plugin: { name, version: String(semver.major(version)) }, pluginId: ... } })`. +- If a processing's plugin id has no matching dir on the volume → fail-fast (throw). Operator must restore the volume or delete the orphaned processing manually. +- If `dataDir` is unset AND there are any legacy-string processings → fail-fast at startup with a clear message ("set DATA_DIR and re-mount the plugins volume for the v6.0 boot"). + +### E. Dev / test environment + +#### E.1 docker-compose +Add to `docker-compose.yml`: + +```yaml +registry: + profiles: [dev] + image: ghcr.io/data-fair/registry:main + network_mode: host + depends_on: [mongo, simple-directory] + environment: + - PORT=${REGISTRY_PORT} + - MONGO_URL=mongodb://localhost:${MONGO_PORT}/data-fair-registry + - PRIVATE_DIRECTORY_URL=http://localhost:${SD_PORT} + - PUBLIC_URL=http://${DEV_HOST}:${NGINX_PORT1}/registry + - SECRET_INTERNAL_SERVICES=secret-registry-internal + - OBSERVER_ACTIVE=false + - STORAGE_TYPE=fs + volumes: + - registry-data:/app/data + +volumes: + registry-data: +``` + +- Add `REGISTRY_PORT` to `dev/init-env.sh` and `.env`. +- Add `/registry/` location in `dev/resources/nginx.conf.template` proxying to `localhost:${REGISTRY_PORT}`. +- `api/config/development.mjs` and `worker/config/development.mjs` add a `registry` block with the dev URL/secret and `cacheDir: '../data/registry-cache'`. + +#### E.2 Test fixtures +- New helper `tests/support/registry.ts` with `publishFixturePlugin(tgzPath, { architecture, public, privateAccess })` that POSTs to registry with `x-secret-key`. +- In `tests/state-setup.ts`, after the existing `DELETE /api/v1/test-env` cleanup, add a registry cleanup endpoint call (or directly clear the registry's mongo `data-fair-registry` db). +- `tests/fixtures/processing-hello-world.tgz` — kept and reused as the canonical fixture, published to registry by `publishFixturePlugin` in tests that need it. + +#### E.3 Tests that change +- Delete `tests/features/plugins/install.api.spec.ts` and `tests/features/plugins/registry.api.spec.ts`. +- `tests/features/processings/lifecycle.api.spec.ts` — every `processing.plugin` literal becomes `{ name, version }`. Setup pre-publishes the fixture plugin and grants access to the test org via `POST /api/v1/access-grants`. +- `tests/features/processings/permissions.api.spec.ts` — same plugin-shape rewrite + ensure registry grants exist for the orgs each test exercises (or that the artefact is `public:true`). +- New `tests/features/registry-integration/migration.api.spec.ts` — pre-seeds a `dataDir/plugins/...` fixture, runs the upgrade scripts, asserts the artefact + grants now exist in registry and the corresponding `processings` rows are rewritten. + +### F. Deprecation timeline +- **v6.0** (this work): legacy plugins volume optional, read-only for plugin-config injection. First-boot migration runs idempotently. UI plugin admin gone. Registry is source of truth for everything else. +- **v6.x patches**: monitor "deprecation: plugin used pluginConfig" warnings in worker logs; encourage plugin authors to publish updated versions that don't read it. +- **v7.0**: drop `config.dataDir`, the `if (config.dataDir)` block in worker `task.ts`, the `legacyIdFor()` helper, the `upgrade/6.0.0/` scripts, and the volume mount in deploy manifests / docker-compose. + +## Critical files to change + +Registry (`/home/alban/data-fair/registry`): +- `api/src/artefacts/service-pure.ts` +- `api/src/artefacts/router.ts` +- `api/src/auth.ts` +- `api/src/access.ts` +- `api/types/artefact/schema.js` +- `lib-node/index.ts`, `lib-node/index.d.ts` + +Processings (`/home/alban/data-fair/processings_feat-registry`): +- `worker/src/task/task.ts` +- `api/src/processings/router.ts` +- `api/src/app.ts` +- `api/types/processing/schema.js` +- `api/config/default.mjs`, `worker/config/default.mjs`, `*/config/custom-environment-variables.mjs` +- `api/src/config.ts` (uiConfig) +- `ui/src/pages/processings/new.vue` (or wherever the plugin picker is) +- `docker-compose.yml`, `dev/init-env.sh`, `dev/resources/nginx.conf.template` +- New: `upgrade/6.0.0/01-publish-plugins-to-registry.ts`, `02-push-metadata-and-access.ts`, `03-rewrite-processing-plugin.ts` +- New: `tests/support/registry.ts`, `tests/features/registry-integration/migration.api.spec.ts` + +To delete: +- `api/src/plugins/router.ts`, `api/src/plugins-registry/router.ts`, `api/types/plugin/` +- `ui/src/pages/admin/plugins.vue` +- `tests/features/plugins/install.api.spec.ts`, `tests/features/plugins/registry.api.spec.ts` + +## Verification + +### End-to-end manual test +1. `docker compose --profile dev up -d` — brings up registry alongside processings, mongo, simple-directory. +2. As superadmin in registry UI, create an upload API key. +3. Build a self-contained plugin tarball (`npm pack` over a directory that already has `node_modules`) and `curl -F file=@plugin.tgz -F architecture=x64 -H 'x-api-key: ...' http://localhost/registry/api/v1/artefacts/@data-fair%2Fprocessing-hello-world/versions`. +4. PATCH the artefact `public: true` (or grant access to `organization:test_org1`). +5. As `test_admin1@test.com` in processings UI, navigate to `/processings/new` — registry plugin list renders. +6. Pick the plugin, fill config, save → API runs `validateFullProcessing` (HTTP fetch) and `prepareProcessing` (lib-node download to API cache). Both succeed. +7. Trigger a run. Worker downloads via lib-node into its own cache (`downloaded: true` first time, `false` on the next run). +8. Smoke checks: + - Revoke org's grant in registry → next run fails with a clear 403 from `ensureArtefact`. + - Upload only `arm64` of a new version on an `x64` worker → 404. Re-upload without `architecture` → noarch fallback works on both arches. + +### Tests that must pass +- `tests/features/processings/lifecycle.api.spec.ts` (rewritten) +- `tests/features/processings/permissions.api.spec.ts` (rewritten) +- New `tests/features/registry-integration/migration.api.spec.ts` +- Registry: new arch-routing tests, x-account auth tests +- Existing unit tests in `tests/features/worker-utils/`, `api-utils/`, `shared/` (not affected) + +### Smoke matrix +| Scenario | Expected | +|---|---| +| Owner has grant, plugin public, arch matches | 200, run completes | +| Owner has grant, plugin private with grant, arch matches | 200, run completes | +| Owner has grant, plugin private without grant on this org | 403 from `ensureArtefact`, run fails with clear log | +| x64 worker, only arm64 tarball exists | 404 | +| x64 worker, arm64 + noarch tarballs exist | 200, downloads noarch | +| Save with valid config | 200 | +| Save with config violating `processingConfigSchema` | 400 | +| First-boot migration on volume with 3 plugins | all 3 published once, idempotent on re-run | +| First-boot with one plugin already in registry | skipped, no error | +| Migration with orphan `processing.plugin` (dir missing) | startup fails fast with operator-actionable error | +| Migration with distTag plugin dir | warning logged, processing referencing it left as legacy string until manually fixed | diff --git a/package-lock.json b/package-lock.json index c29efe79..dcdb09ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,10 +42,25 @@ "node": "v24" } }, + "../registry/lib-node": { + "name": "@data-fair/lib-node-registry", + "version": "0.1.3", + "license": "MIT", + "dependencies": { + "@types/resolve-path": "^1.4.3", + "@types/tar-stream": "^3.1.4", + "resolve-path": "^1.4.0", + "tar-stream": "^3.1.0" + }, + "peerDependencies": { + "@data-fair/lib-node": ">=2.8.0" + } + }, "api": { "dependencies": { "@data-fair/lib-express": "^1.22.5", "@data-fair/lib-node": "^2.12.1", + "@data-fair/lib-node-registry": "file:../../registry/lib-node", "@data-fair/lib-utils": "^1.10.1", "@data-fair/processings-shared": "*", "ajv": "^8.17.1", @@ -593,6 +608,10 @@ } } }, + "node_modules/@data-fair/lib-node-registry": { + "resolved": "../registry/lib-node", + "link": true + }, "node_modules/@data-fair/lib-types-builder": { "version": "1.11.6", "resolved": "https://registry.npmjs.org/@data-fair/lib-types-builder/-/lib-types-builder-1.11.6.tgz", @@ -1666,6 +1685,27 @@ "vue": "^3.0.0" } }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "license": "ISC", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@isaacs/fs-minipass/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -2773,6 +2813,17 @@ "@types/node": "*" } }, + "node_modules/@types/tar": { + "version": "6.1.13", + "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.13.tgz", + "integrity": "sha512-IznnlmU5f4WcGTh2ltRu/Ijpmk8wiWXfF0VA4s+HPjHZgvFggk1YaIkbo5krX/zUCzWF8N/l4+W/LNxnvAJ8nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "minipass": "^4.0.0" + } + }, "node_modules/@types/web-bluetooth": { "version": "0.0.21", "resolved": "https://registry.npmjs.org/@types/web-bluetooth/-/web-bluetooth-0.0.21.tgz", @@ -4283,6 +4334,15 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/chownr": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -8040,6 +8100,37 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", + "license": "MIT", + "dependencies": { + "minipass": "^7.1.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/minizlib/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/mlly": { "version": "1.8.2", "resolved": "https://registry.npmjs.org/mlly/-/mlly-1.8.2.tgz", @@ -10326,6 +10417,40 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tar": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.2", + "minizlib": "^3.1.0", + "yallist": "^5.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tar/node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/tar/node_modules/yallist": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=18" + } + }, "node_modules/tdigest": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", @@ -11730,6 +11855,7 @@ "worker": { "dependencies": { "@data-fair/lib-node": "^2.12.1", + "@data-fair/lib-node-registry": "file:../../registry/lib-node", "@data-fair/processings-shared": "*", "axios": "^1.8.3", "axios-retry": "^4.5.0", @@ -11739,13 +11865,16 @@ "nodemailer": "^8.0.4", "prom-client": "^15.1.3", "resolve-path": "^1.4.0", + "semver": "^7.7.4", + "tar": "^7.5.1", "tmp-promise": "^3.0.3", "tree-kill": "^1.2.2" }, "devDependencies": { "@data-fair/lib-express": "^1.22.5", "@types/nodemailer": "^6.4.16", - "@types/resolve-path": "^1.4.3" + "@types/resolve-path": "^1.4.3", + "@types/tar": "^6.1.13" } } } diff --git a/shared/plugin-id.ts b/shared/plugin-id.ts new file mode 100644 index 00000000..a5b96cd2 --- /dev/null +++ b/shared/plugin-id.ts @@ -0,0 +1,15 @@ +/** + * `pluginId` is the registry artefact id stored on `processing.pluginId`. + * Shape: `{name}@{major}` where `name` is the npm package name (which itself + * may start with `@scope/`). Split on the LAST `@` so scoped names parse. + */ +export interface ParsedPluginId { + name: string + major: string +} + +export const parsePluginId = (pluginId: string): ParsedPluginId => { + const at = pluginId.lastIndexOf('@') + if (at <= 0) throw new Error(`invalid pluginId "${pluginId}": expected "{name}@{major}"`) + return { name: pluginId.slice(0, at), major: pluginId.slice(at + 1) } +} diff --git a/tests/features/plugins/install.api.spec.ts b/tests/features/plugins/install.api.spec.ts deleted file mode 100644 index 8261005b..00000000 --- a/tests/features/plugins/install.api.spec.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { test, expect } from '@playwright/test' -import { axios, axiosAuth, clean } from '../../support/axios.ts' - -const axAno = axios() - -const plugin = { - name: '@data-fair/processing-hello-world', - version: '1.2.2', - distTag: 'latest', -} -const pluginId = '@data-fair-processing-hello-world-1' - -test.describe('plugin', () => { - test.beforeAll(clean) - test.afterAll(clean) - - test('should install a plugin from npm', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - - const res = await superadmin.post('/api/v1/plugins', { - name: plugin.name, - version: '0.13.0', // Previous version to test update - distTag: 'latest' - }) - expect(res.data.name).toBe(plugin.name) - expect(res.data.id).toBe('@data-fair-processing-hello-world-0') - expect(res.data.version).toBe('0.13.0') - - // Only superadmin can install plugins - await expect(adminTestOrg1.post('/api/v1/plugins')).rejects.toMatchObject({ status: 403 }) - }) - - test('should install a plugin from tarball', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - const FormData = (await import('form-data')).default - const fs = await import('fs') - const path = await import('path') - - const tarballPath = path.join(import.meta.dirname, '..', '..', 'fixtures', 'processing-hello-world.tgz') - const formData = new FormData() - formData.append('file', fs.createReadStream(tarballPath)) - - const res = await superadmin.post('/api/v1/plugins', formData, { - headers: formData.getHeaders() - }) - - expect(res.data.name).toBe(plugin.name) - expect(res.data.id).toBe(pluginId) - expect(res.data.version).toBe('1.2.2') - - await expect(adminTestOrg1.post('/api/v1/plugins')).rejects.toMatchObject({ status: 403 }) - }) - - test('should update a plugin', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - - const res = await superadmin.post('/api/v1/plugins', plugin) - expect(res.data.name).toBe(plugin.name) - expect(res.data.id).toBe(pluginId) - expect(res.data.version).toBe(plugin.version) - - await expect(adminTestOrg1.post('/api/v1/plugins')).rejects.toMatchObject({ status: 403 }) - }) - - test('should list installed plugins', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - - let res = await superadmin.get('/api/v1/plugins') - expect(res.data.count).toBe(2) - expect(res.data.results.length).toBe(2) - expect(res.data.results[0].name).toBe(plugin.name) - - await expect(adminTestOrg1.get('/api/v1/plugins')).rejects.toMatchObject({ status: 400 }) - - res = await adminTestOrg1.get('/api/v1/plugins?privateAccess=organization:test_org1') - expect(res.data.count).toBe(0) - expect(res.data.results.length).toBe(0) - }) - - test('should get specific plugin', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - - await expect(superadmin.get('/api/v1/plugins/does-not-exist')).rejects.toMatchObject({ status: 404 }) - - const res = await superadmin.get('/api/v1/plugins/' + pluginId) - expect(res.data.name).toBe(plugin.name) - expect(res.data.id).toBe(pluginId) - expect(res.data.version).toBe(plugin.version) - }) - - test('should manage plugin access permissions', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - const aloneUser = await axiosAuth('test_alone@test.com') - - // make the plugin private with specific access to admin only - await superadmin.put(`/api/v1/plugins/${pluginId}/access`, { - public: false, - privateAccess: [{ type: 'organization', id: 'test_org1' }] - }) - - let res = await adminTestOrg1.get('/api/v1/plugins?privateAccess=organization:test_org1') - expect(res.data.results.length).toBe(1) - - res = await aloneUser.get('/api/v1/plugins?privateAccess=user:test_alone') - expect(res.data.results.length).toBe(0) - - await superadmin.put(`/api/v1/plugins/${pluginId}/access`, { public: true }) - - res = await adminTestOrg1.get('/api/v1/plugins?privateAccess=organization:test_org1') - expect(res.data.results.length).toBe(1) - - res = await aloneUser.get('/api/v1/plugins?privateAccess=user:test_alone') - expect(res.data.results.length).toBe(1) - }) - - test('should delete a plugin', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) - - let res = await superadmin.get('/api/v1/plugins') - expect(res.data.count).toBe(2) - - await expect(axAno.delete(`/api/v1/plugins/${pluginId}`)).rejects.toMatchObject({ status: 401 }) - await expect(adminTestOrg1.delete(`/api/v1/plugins/${pluginId}`)).rejects.toMatchObject({ status: 403 }) - - res = await superadmin.delete(`/api/v1/plugins/${pluginId}`) - expect(res.status).toBe(204) - - await expect(superadmin.get('/api/v1/plugins/' + pluginId)).rejects.toMatchObject({ status: 404 }) - await expect(superadmin.delete(`/api/v1/plugins/${pluginId}`)).rejects.toMatchObject({ status: 404 }) - }) -}) diff --git a/tests/features/plugins/registry.api.spec.ts b/tests/features/plugins/registry.api.spec.ts deleted file mode 100644 index 9591949d..00000000 --- a/tests/features/plugins/registry.api.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { test, expect } from '@playwright/test' -import { axios } from '../../support/axios.ts' - -const axAno = axios() - -test.describe('plugin-registry', () => { - test('should search for plugins (just latest) on npmjs', async () => { - const res = await axAno.get('/api/v1/plugins-registry', { params: { q: 'hello-world' } }) - const hwProcessingPackages = res.data.results.filter((p: { name: string }) => p.name === '@data-fair/processing-hello-world') - expect(hwProcessingPackages.length).toBe(1) - expect(res.data.results[0].distTag).toBe('latest') - }) - - test('should search for plugins and all their distTag versions on npmjs', async () => { - const res = await axAno.get('/api/v1/plugins-registry', { params: { q: 'hello-world', showAll: 'true' } }) - const hwProcessingPackages = res.data.results.filter((p: { name: string }) => p.name === '@data-fair/processing-hello-world') - expect(hwProcessingPackages.length).toBe(2) - expect(['latest', 'test']).toContain(res.data.results[0].distTag) - expect(['latest', 'test']).toContain(res.data.results[1].distTag) - }) -}) diff --git a/tests/features/processings/lifecycle.api.spec.ts b/tests/features/processings/lifecycle.api.spec.ts index 74635804..646283da 100644 --- a/tests/features/processings/lifecycle.api.spec.ts +++ b/tests/features/processings/lifecycle.api.spec.ts @@ -1,16 +1,16 @@ import { test, expect } from '@playwright/test' import { axiosAuth, clean, waitForRunStatus } from '../../support/axios.ts' - -const installTestPlugin = async (superadmin: any) => { - const plugin = (await superadmin.post('/api/v1/plugins', { - name: '@data-fair/processing-hello-world', - version: '1.2.2', - distTag: 'latest', - description: 'Minimal plugin for data-fair-processings. Create one-line datasets on demand.' - })).data - await superadmin.put(`/api/v1/plugins/${plugin.id}/access`, { public: true }) - return plugin -} +import { publishFixturePlugin } from '../../support/registry.ts' + +// The hello-world fixture in tests/fixtures was packed by `npm pack` and does +// NOT carry node_modules. Tests that only assert on the API surface (create, +// patch, list, validate) work against it as-is. Tests that trigger an actual +// run depend on the plugin module being importable; those are now expected +// to fail until we ship a runnable bundle (or rebuild one in beforeAll). +const installTestPlugin = async () => publishFixturePlugin({ + name: '@data-fair/processing-hello-world', + version: '1.2.2' +}) test.describe('processing', () => { test.beforeEach(clean) @@ -18,11 +18,11 @@ test.describe('processing', () => { test('should create a new processing, activate it and run it', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() let processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' } })).data expect(processing._id).toBeTruthy() @@ -74,11 +74,11 @@ test.describe('processing', () => { test('should kill a long run with SIGTERM', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' }, active: true, config: { @@ -102,11 +102,11 @@ test.describe('processing', () => { test('should kill a long run with SIGTERM and wait for grace period', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' }, active: true, config: { @@ -128,7 +128,7 @@ test.describe('processing', () => { test('should fail a run if processings_seconds limit is exceeded', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() await superadmin.post('/api/v1/limits/user/test_superadmin', { processings_seconds: { limit: 1 }, @@ -137,7 +137,7 @@ test.describe('processing', () => { const processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' }, active: true, config: { @@ -163,13 +163,12 @@ test.describe('processing', () => { }) test('should manage a processing as a department admin', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const depAdmin = await axiosAuth({ email: 'test_dep_admin@test.com', org: 'test_org1', dep: 'dep1' }) const processing = (await depAdmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id + pluginId: plugin.pluginId })).data const processings = (await depAdmin.get('/api/v1/processings')).data @@ -196,11 +195,11 @@ test.describe('processing', () => { test('should config a new processing with a secret field', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' } })).data expect(processing._id).toBeTruthy() @@ -241,11 +240,11 @@ test.describe('processing', () => { test('should patch config with secrets', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const processing = (await superadmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' }, active: true, config: { diff --git a/tests/features/processings/permissions.api.spec.ts b/tests/features/processings/permissions.api.spec.ts index ce3f98e3..05c84a87 100644 --- a/tests/features/processings/permissions.api.spec.ts +++ b/tests/features/processings/permissions.api.spec.ts @@ -1,24 +1,18 @@ import { test, expect } from '@playwright/test' import { axiosAuth, clean } from '../../support/axios.ts' +import { publishFixturePlugin } from '../../support/registry.ts' -const installTestPlugin = async (superadmin: any) => { - const plugin = (await superadmin.post('/api/v1/plugins', { - name: '@data-fair/processing-hello-world', - version: '0.12.2', - distTag: 'latest', - description: 'Minimal plugin for data-fair-processings. Create one-line datasets on demand.' - })).data - await superadmin.put(`/api/v1/plugins/${plugin.id}/access`, { public: true }) - return plugin -} +const installTestPlugin = async () => publishFixturePlugin({ + name: '@data-fair/processing-hello-world', + version: '1.2.2' +}) test.describe('processing permissions', () => { test.beforeEach(clean) test.afterAll(clean) test('should create a processing and check permissions across roles', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) const contribTestOrg1 = await axiosAuth({ email: 'test_contrib1@test.com', org: 'test_org1' }) @@ -28,7 +22,7 @@ test.describe('processing permissions', () => { const processing = (await adminTestOrg1.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id + pluginId: plugin.pluginId })).data await adminTestOrg1.patch(`/api/v1/processings/${processing._id}`, { @@ -95,23 +89,23 @@ test.describe('processing permissions', () => { await expect(partnerAdmin.patch(`/api/v1/processings/${processing._id}`, { title: 'test' })).rejects.toMatchObject({ status: 403 }) await expect(aloneOutsider.patch(`/api/v1/processings/${processing._id}`, { title: 'test' })).rejects.toMatchObject({ status: 403 }) - await superadmin.delete(`/api/v1/plugins/${plugin.id}`) + // plugin lifecycle is managed by registry now — no per-test deletion }) test('should list processings with proper org isolation', async () => { const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const adminTestOrg1 = await axiosAuth({ email: 'test_admin1@test.com', org: 'test_org1' }) const partnerAdmin = await axiosAuth({ email: 'test_user2@test.com', org: 'test_org2' }) await adminTestOrg1.post('/api/v1/processings', { title: 'Hello processing 1', - plugin: plugin.id + pluginId: plugin.pluginId }) await partnerAdmin.post('/api/v1/processings', { title: 'Hello processing 2', - plugin: plugin.id + pluginId: plugin.pluginId }) expect((await adminTestOrg1.get('/api/v1/processings?owner=organization:test_org1')).data.count).toBe(1) @@ -122,14 +116,13 @@ test.describe('processing permissions', () => { }) test('should manage processings as a department admin', async () => { - const superadmin = await axiosAuth('test_superadmin@test.com') - const plugin = await installTestPlugin(superadmin) + const plugin = await installTestPlugin() const depAdmin = await axiosAuth({ email: 'test_dep_admin@test.com', org: 'test_org1', dep: 'dep1' }) // create a processing in his department const processing = (await depAdmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { id: 'test_org1', name: 'Test Org 1', @@ -143,7 +136,7 @@ test.describe('processing permissions', () => { // cannot create in another department await expect(depAdmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { id: 'test_org1', name: 'Test Org 1', @@ -156,7 +149,7 @@ test.describe('processing permissions', () => { // cannot create in the root organization (no department) await expect(depAdmin.post('/api/v1/processings', { title: 'Hello processing', - plugin: plugin.id, + pluginId: plugin.pluginId, owner: { id: 'test_org1', name: 'Test Org 1', diff --git a/tests/features/ui/layout.e2e.spec.ts b/tests/features/ui/layout.e2e.spec.ts index 47026ea6..513b616c 100644 --- a/tests/features/ui/layout.e2e.spec.ts +++ b/tests/features/ui/layout.e2e.spec.ts @@ -1,8 +1,6 @@ -import path from 'node:path' -import fs from 'node:fs' -import FormData from 'form-data' import { test, expect } from '../../fixtures/login.ts' import { axiosAuth, clean } from '../../support/axios.ts' +import { publishFixturePlugin } from '../../support/registry.ts' test.describe('UI layout', () => { test.beforeEach(clean) @@ -15,16 +13,14 @@ test.describe('UI layout', () => { test('processings list renders a card for an existing processing', async ({ page, goToWithAuth }) => { const superadmin = await axiosAuth('test_superadmin@test.com') - - const tarballPath = path.join(import.meta.dirname, '..', '..', 'fixtures', 'processing-hello-world.tgz') - const formData = new FormData() - formData.append('file', fs.createReadStream(tarballPath)) - const plugin = (await superadmin.post('/api/v1/plugins', formData, { headers: formData.getHeaders() })).data - await superadmin.put(`/api/v1/plugins/${plugin.id}/access`, { public: true }) + const fixture = await publishFixturePlugin({ + name: '@data-fair/processing-hello-world', + version: '1.2.2' + }) await superadmin.post('/api/v1/processings', { title: 'My e2e processing', - plugin: plugin.id, + pluginId: fixture.pluginId, owner: { type: 'user', id: 'test_superadmin', name: 'Test Super Admin' } }) diff --git a/tests/support/axios.ts b/tests/support/axios.ts index dbf29bea..4c0e12e9 100644 --- a/tests/support/axios.ts +++ b/tests/support/axios.ts @@ -43,6 +43,10 @@ export const clean = async () => { await waitForWorkerIdle() await anonymousAx.delete(`${apiUrl}/api/v1/test-env`) await anonymousAx.delete(`${apiUrl}/api/v1/test-env/plugins`) + // Registry has no test-env endpoint — drop the db directly. Lazy import to + // avoid pulling the mongodb client into specs that don't need it. + const { cleanRegistryDb } = await import('./registry.ts') + await cleanRegistryDb() } /** Poll the test-env raw-run endpoint until status matches one of the given values. */ diff --git a/tests/support/registry.ts b/tests/support/registry.ts new file mode 100644 index 00000000..7c945dd7 --- /dev/null +++ b/tests/support/registry.ts @@ -0,0 +1,108 @@ +import fs from 'node:fs' +import path from 'node:path' +import FormData from 'form-data' +import { MongoClient } from 'mongodb' +import { axiosBuilder } from '@data-fair/lib-node/axios.js' + +/** + * Registry test harness — talks to the dev-mode registry container at + * /registry on the same nginx as processings, with the matching internal + * secret defined in `docker-compose.yml`. + * + * Tests publish their fixture plugin once at the top of a describe block (or + * via state-setup) and reset registry state between runs by dropping the + * `data-fair-registry` mongo db directly. + */ + +export const registryBaseUrl = `http://${process.env.DEV_HOST}:${process.env.NGINX_PORT1}/registry` +export const registryInternalSecret = 'secret-registry-internal' + +export const axiosRegistryInternal = axiosBuilder({ + baseURL: registryBaseUrl, + headers: { 'x-secret-key': registryInternalSecret } +}) + +export interface PublishedFixture { + /** Registry artefact id (`{name}@{major}`) — same value stored on `processing.pluginId`. */ + pluginId: string +} + +export interface PublishOptions { + /** npm package name as it appears in package.json — used in the upload URL. */ + name: string + /** Full semver of the version being uploaded (e.g. `1.2.3`). */ + version: string + /** Defaults to the host architecture. */ + architecture?: string + /** Local path to the .tgz to upload. */ + tarballPath?: string + /** Set the artefact public (default true) — the cheap path for most tests. */ + isPublic?: boolean + /** Per-account grants to seed in lieu of, or alongside, public. */ + privateAccess?: { type: 'user' | 'organization', id: string }[] +} + +const DEFAULT_TARBALL = path.resolve(import.meta.dirname, '../fixtures/processing-hello-world.tgz') + +/** + * Publish a fixture plugin to registry as the internal service. Patches the + * artefact metadata so it is visible to the test owner, and returns the + * pluginId string that `processing.pluginId` now expects. + * + * NOTE: the bundled `processing-hello-world.tgz` fixture is the plain + * `npm pack` output and does NOT include `node_modules`. That is fine for + * tests that only exercise the API surface (validate, prepare list, picker). + * Tests that actually trigger a run need a runnable bundle — generate one in + * a beforeAll (npm install + repack into a tmp tarball) or assert on + * non-execution behaviour. + */ +export const publishFixturePlugin = async (opts: PublishOptions): Promise => { + const tarballPath = opts.tarballPath ?? DEFAULT_TARBALL + const tgz = await fs.promises.readFile(tarballPath) + const arch = opts.architecture ?? process.arch + + const form = new FormData() + form.append('architecture', arch) + form.append('file', tgz, { filename: 'package.tgz', contentType: 'application/gzip' }) + await axiosRegistryInternal.post( + `/api/v1/artefacts/${encodeURIComponent(opts.name)}/versions`, + form, + { headers: form.getHeaders() } + ) + + const major = parseInt(opts.version.split('.')[0], 10) + const pluginId = `${opts.name}@${major}` + + // Artefacts are keyed by package name; the major suffix only lives on + // the processings side as a runtime version pin. + await axiosRegistryInternal.patch(`/api/v1/artefacts/${encodeURIComponent(opts.name)}`, { + public: opts.isPublic ?? true, + privateAccess: opts.privateAccess ?? [] + }) + + for (const acc of opts.privateAccess ?? []) { + await axiosRegistryInternal.post( + '/api/v1/access-grants', + { account: acc }, + { validateStatus: s => s === 201 || s === 409 } + ) + } + + return { pluginId } +} + +/** + * Drop the registry mongo db. Called from the global state-setup before each + * test run to ensure a clean registry across runs. Uses the direct mongo + * connection because registry has no test-env DELETE endpoint. + */ +export const cleanRegistryDb = async () => { + const url = `mongodb://localhost:${process.env.MONGO_PORT ?? '27017'}/data-fair-registry` + const client = new MongoClient(url) + try { + await client.connect() + await client.db().dropDatabase() + } finally { + await client.close() + } +} diff --git a/ui/src/components/processing/processing-actions.vue b/ui/src/components/processing/processing-actions.vue index 0ea02ea1..77390f4f 100644 --- a/ui/src/components/processing/processing-actions.vue +++ b/ui/src/components/processing/processing-actions.vue @@ -248,8 +248,8 @@ @@ -316,11 +316,11 @@ import '@data-fair/frame/lib/d-frame.js' const emit = defineEmits(['triggered']) -const { canAdmin, canExec, edited, metadata, processing, processingSchema } = defineProps<{ +const { canAdmin, canExec, edited, documentation, processing, processingSchema } = defineProps<{ canAdmin: boolean, canExec: boolean, edited: boolean, - metadata: Record | undefined, + documentation?: string, processing: Record, processingSchema: Record, }>() @@ -367,7 +367,7 @@ const confirmDuplicate = useAsyncAction( const newProcessing = { owner: processing.owner, - plugin: processing.plugin, + pluginId: processing.pluginId, title: duplicateTitle.value || `${processing.title} ${t('copy')}`, config: processing.config, permissions: processing.permissions, diff --git a/ui/src/components/processing/processing-card.vue b/ui/src/components/processing/processing-card.vue index 3e586262..c6ae2eb1 100644 --- a/ui/src/components/processing/processing-card.vue +++ b/ui/src/components/processing/processing-card.vue @@ -45,14 +45,14 @@ /> - {{ t('deleted') + ' - ' + processing.plugin }} + {{ t('deleted') + ' - ' + processing.pluginId }} - {{ pluginFetch.data.value?.metadata.name }} + {{ pluginFetch.data.value?.title?.fr ?? pluginFetch.data.value?.title?.en ?? pluginFetch.data.value?.name ?? processing.pluginId }} @@ -186,7 +186,7 @@ const props = defineProps({ showOwner: Boolean }) -const pluginFetch = usePluginFetch(props.processing.plugin) +const pluginFetch = usePluginFetch(props.processing.pluginId) diff --git a/ui/src/components/processings-actions.vue b/ui/src/components/processings-actions.vue index 2fa4dff6..a0179972 100644 --- a/ui/src/components/processings-actions.vue +++ b/ui/src/components/processings-actions.vue @@ -130,23 +130,18 @@ const statusesText = computed>(() => ({ triggered: t('statusTriggered') })) -type InstalledPlugin = { +// Subset of registry's Artefact shape used to render the plugin filter labels. +type RegistryArtefact = { + _id: string name: string - description: string - version: string - distTag: string - id: string - pluginConfigSchema: any - processingConfigSchema: any - metadata: { - name: string - description: string - category: string - icon: Record - } + majorVersion: number + title?: { fr?: string, en?: string } } -const installedPluginsFetch = useFetch<{ results: InstalledPlugin[], count: number }>(`${$apiPath}/plugins?privateAccess=${processingsProps.ownerFilter}`) +const installedPluginsFetch = useFetch<{ results: RegistryArtefact[], count: number }>( + '/registry/api/v1/artefacts', + { query: { category: 'processing', size: 200 } } +) const installedPlugins = computed(() => installedPluginsFetch.data.value?.results) const eventsSubscribeUrl = computed(() => { @@ -175,10 +170,13 @@ const pluginsItems = computed(() => { if (!installedPlugins.value) return [] if (!processingsProps.facets.plugins) return [] + // facets.plugins keys are the denormalized pluginId — `${name}@${major}` — + // matching registry artefact._id directly. return Object.entries(processingsProps.facets.plugins) .map( ([pluginKey, count]) => { - const customName = installedPlugins.value?.find((plugin) => plugin.id === pluginKey)?.metadata.name + const artefact = installedPlugins.value?.find((p) => p._id === pluginKey) + const customName = artefact?.title?.fr ?? artefact?.title?.en ?? artefact?.name return { display: `${customName || t('deleted') + ' - ' + pluginKey} (${count})`, pluginKey diff --git a/ui/src/composables/use-plugin-fetch.ts b/ui/src/composables/use-plugin-fetch.ts index faba6c24..2b6d8d3c 100644 --- a/ui/src/composables/use-plugin-fetch.ts +++ b/ui/src/composables/use-plugin-fetch.ts @@ -1,12 +1,36 @@ -import type { Plugin } from '#api/types' +import { parsePluginId } from '@data-fair/processings-shared/plugin-id.ts' -const fetches: Record>> = {} +// Subset of registry's Artefact shape that the UI uses. Artefacts are now +// keyed by package name (no @major suffix); per-major data — including +// processingConfigSchema — lives on the version documents instead. +export interface RegistryArtefact { + _id: string + name: string + latestMajor?: number + category: string + title?: { fr?: string, en?: string } + description?: { fr?: string, en?: string } + group?: { fr?: string, en?: string } + documentation?: string + thumbnail?: { id: string, width: number, height: number } +} + +const fetches: Record>> = {} +/** + * Fetch artefact metadata from the registry for a processing's pluginId + * (`{name}@{major}`). Only the `name` part identifies the artefact; the + * major is the runtime version pin and is not used here. + * + * Same-domain assumption: registry is always mounted at `/registry` of the + * current domain. The session cookie is sent automatically. + */ export const usePluginFetch = (pluginId: string) => { - if (!fetches[pluginId]) { - const pluginFetch = useFetch(`${$apiPath}/plugins/${pluginId}`) - fetches[pluginId] = pluginFetch + const { name } = parsePluginId(pluginId) + if (!fetches[name]) { + fetches[name] = useFetch(`/registry/api/v1/artefacts/${encodeURIComponent(name)}`) } - return fetches[pluginId] + return fetches[name] } + export default usePluginFetch diff --git a/ui/src/pages/admin/index.vue b/ui/src/pages/admin/index.vue deleted file mode 100644 index fc59f41e..00000000 --- a/ui/src/pages/admin/index.vue +++ /dev/null @@ -1,9 +0,0 @@ - - - diff --git a/ui/src/pages/admin/plugins.vue b/ui/src/pages/admin/plugins.vue deleted file mode 100644 index 73dc1c02..00000000 --- a/ui/src/pages/admin/plugins.vue +++ /dev/null @@ -1,675 +0,0 @@ - - - - - -en: - adminRequired: You do not have permission to access this page, you need to enable super-admin mode. - authRequired: Authentication required - availablePluginsLabel: available plugins - cancel: Cancel - install: Install - installError: Error while installing the plugin - installSeparately: Install separately - installSuccess: Plugin installed! - installedPluginsLabel: installed plugins - majorInstallBody: Installing a major version will create a separate entry. Existing processings will continue to use the current version. - majorInstallTitleWithVersions: Major version installation - manualInstall: Install a plugin manually - manualInstallDistTag: Distribution tag - manualInstallFileLabel: Select a .tgz file - manualInstallFromFile: Install from a file - manualInstallFromNpm: Install from npm - manualInstallName: Plugin name - manualInstallTitle: Manual plugin installation - manualInstallVersion: Plugin version - no: No - pluginsBreadcrumb: Plugins - showAllLabel: Show test versions of plugins - uninstall: Uninstall - uninstallConfirm: Are you sure you want to uninstall the plugin "{name}"? - uninstallError: Error while uninstalling the plugin - uninstallSuccess: Plugin uninstalled! - uninstallTitle: Uninstall plugin - update: Update - updateWithVersion: Update ({version}) - usageCount: Used {count} times - yes: Yes - -fr: - adminRequired: Vous n'avez pas la permission d'accéder à cette page, il faut avoir activé le mode super-administration. - authRequired: Authentification nécessaire - availablePluginsLabel: plugins disponibles - cancel: Annuler - install: Installer - installError: Erreur lors de l'installation du plugin - installSeparately: Installer séparément - installSuccess: Plugin installé ! - installedPluginsLabel: plugins installés - majorInstallBody: L'installation d'une version majeure créera une entrée séparée. Les traitements existants continueront d'utiliser la version actuelle. - majorInstallTitleWithVersions: Installation d'une version majeure - manualInstall: Installer un plugin manuellement - manualInstallDistTag: Tag de distribution - manualInstallFileLabel: Sélectionner un fichier .tgz - manualInstallFromFile: Installer depuis un fichier - manualInstallFromNpm: Installer depuis npm - manualInstallName: Nom du plugin - manualInstallTitle: Installation manuelle d'un plugin - manualInstallVersion: Version du plugin - no: Non - pluginsBreadcrumb: Plugins - showAllLabel: Afficher les versions de test des plugins - uninstall: Désinstaller - uninstallConfirm: Voulez-vous vraiment désinstaller le plugin "{name}" ? - uninstallError: Erreur lors de la désinstallation du plugin - uninstallSuccess: Plugin désinstallé ! - uninstallTitle: Désinstallation du plugin - update: Mettre à jour - updateWithVersion: Mettre à jour ({version}) - usageCount: Utilisé {count} fois - yes: Oui - diff --git a/ui/src/pages/dev.vue b/ui/src/pages/dev.vue index 2ba33140..ab47df94 100644 --- a/ui/src/pages/dev.vue +++ b/ui/src/pages/dev.vue @@ -7,13 +7,6 @@ > Traitements - - Plugins - diff --git a/ui/src/components/processings-actions.vue b/ui/src/components/processings-actions.vue index a0179972..bc27df83 100644 --- a/ui/src/components/processings-actions.vue +++ b/ui/src/components/processings-actions.vue @@ -170,7 +170,7 @@ const pluginsItems = computed(() => { if (!installedPlugins.value) return [] if (!processingsProps.facets.plugins) return [] - // facets.plugins keys are the denormalized pluginId — `${name}@${major}` — + // facets.plugins keys are the registry artefact id stored on `plugin`, // matching registry artefact._id directly. return Object.entries(processingsProps.facets.plugins) .map( diff --git a/ui/src/composables/use-plugin-fetch.ts b/ui/src/composables/use-plugin-fetch.ts index 6eae84e6..c64b0308 100644 --- a/ui/src/composables/use-plugin-fetch.ts +++ b/ui/src/composables/use-plugin-fetch.ts @@ -1,18 +1,9 @@ -import { parsePluginId } from '@data-fair/processings-shared/plugin-id.ts' - -// Subset of registry's Artefact shape that the UI uses. Two flavours share -// the same shape: -// - `npm` artefacts are keyed by package name; per-major data -// (processingConfigSchema, etc.) lives on the version documents. -// - `branch` artefacts have no version history — a single mutable tarball -// sits directly on the artefact doc, replaced on each upload. `branchName` -// is optional metadata (the source git branch). +// Subset of registry's Artefact shape that the UI uses. The artefact `_id` is +// exactly what we store on `processing.plugin`, so we fetch the artefact by that. export interface RegistryArtefact { _id: string name: string - format?: 'npm' | 'file' | 'branch' - latestMajor?: number - branchName?: string + format?: 'npm' | 'file' version?: string category: string title?: { fr?: string, en?: string } @@ -25,9 +16,8 @@ export interface RegistryArtefact { const fetches: Record>> = {} /** - * Fetch artefact metadata from the registry for a processing's pluginId - * (`{name}@{major}`). Only the `name` part identifies the artefact; the - * major is the runtime version pin and is not used here. + * Fetch artefact metadata from the registry for a processing's `plugin` + * (the registry artefact id). * * Errors (404 deleted, 403 no access) are NOT broadcast as a global ui * notification — callers read `error.value` and render their own inline @@ -37,14 +27,13 @@ const fetches: Record>> = { * current domain. The session cookie is sent automatically. */ export const usePluginFetch = (pluginId: string) => { - const { name } = parsePluginId(pluginId) - if (!fetches[name]) { - fetches[name] = useFetch( - `/registry/api/v1/artefacts/${encodeURIComponent(name)}`, + if (!fetches[pluginId]) { + fetches[pluginId] = useFetch( + `/registry/api/v1/artefacts/${encodeURIComponent(pluginId)}`, { notifError: false } ) } - return fetches[name] + return fetches[pluginId] } export default usePluginFetch diff --git a/ui/src/pages/processings/[id]/index.vue b/ui/src/pages/processings/[id]/index.vue index c5d89525..115dcaca 100644 --- a/ui/src/pages/processings/[id]/index.vue +++ b/ui/src/pages/processings/[id]/index.vue @@ -12,20 +12,10 @@ > {{ t('pluginUnavailableBody') }}
- {{ processing?.pluginId }} + {{ processing?.plugin }}

{{ t('processingTitle', { title: processing.title }) }} - - {{ plugin.branchName ? `dev: ${plugin.branchName}` : 'dev build' }} -

@@ -23,7 +23,7 @@ :title="t('information')" value="2" :color="step === '2' ? 'primary' : ''" - :editable="!!newProcessing.pluginId" + :editable="!!newProcessing.plugin" /> @@ -65,15 +65,6 @@ {{ artefactDisplayName(artefact) }} - - {{ artefact.branchName ? `dev: ${artefact.branchName}` : 'dev build' }} -