feat!: v6 registry integration — serve plugins from @data-fair/registry

api/config/custom-environment-variables.mjs

@@ -13,10 +13,12 @@ export default {
   port: 'PORT',
   privateDirectoryUrl: 'PRIVATE_DIRECTORY_URL',
   privateEventsUrl: 'PRIVATE_EVENTS_URL',
+  privateRegistryUrl: 'PRIVATE_REGISTRY_URL',
   secretKeys: {
     limits: 'SECRET_LIMITS',
     events: 'SECRET_EVENTS',
-    identities: 'SECRET_IDENTITIES'
+    identities: 'SECRET_IDENTITIES',
+    registry: 'SECRET_REGISTRY'
   },
   observer: {
     active: 'OBSERVER_ACTIVE',

api/config/default.mjs

@@ -1,7 +1,10 @@
 export default {
   cipherPassword: undefined,
-  dataDir: '/app/data',
-  tmpDir: null, // will be dataDir + '/tmp' if null
+  // Optional. When set, the legacy plugins volume at <dataDir>/plugins is read
+  // by the v6.0 boot migration. Drops with v7.0.
+  dataDir: null,
+  // Defaults to <dataDir>/tmp when dataDir is set, else <os.tmpdir>/data-fair-processings.
+  tmpDir: null,
   defaultLimits: {
     // Maximum time spent running processings
     // -1 for unlimited storage
@@ -10,10 +13,16 @@ export default {
   pluginCategories: ['Essentiels', 'Mes plugins', 'Données de références', 'Tests'],
   privateDirectoryUrl: 'http://simple-directory:8080',
   privateEventsUrl: undefined,
+  // Internal URL the API uses for server-to-server calls to registry. The UI
+  // talks to /registry on the same domain (no public URL is configurable).
+  privateRegistryUrl: 'http://registry:8080',
   secretKeys: {
     limits: null,
     events: undefined,
-    identities: undefined
+    identities: undefined,
+    // x-secret-key shared with registry. Required for save-time validation,
+    // prepare-hook downloads, and the v6.0 first-boot migration.
+    registry: undefined
   },
   mongoUrl: 'mongodb://localhost:27017/data-fair-processings',
   port: 8080,

api/config/development.mjs

@@ -2,6 +2,7 @@ const apiPort = parseInt(process.env.DEV_API_PORT ?? '8082')
 const mongoPort = process.env.MONGO_PORT ?? '27017'
 const sdPort = process.env.SD_PORT ?? '8080'
 const eventsPort = process.env.EVENTS_PORT ?? '8083'
+const registryPort = process.env.REGISTRY_PORT ?? '8085'
 const observerPort = parseInt(process.env.DEV_API_OBSERVER_PORT ?? '9092')
 
 export default {
@@ -14,8 +15,10 @@ export default {
   port: apiPort,
   privateDirectoryUrl: `http://localhost:${sdPort}`,
   privateEventsUrl: `http://localhost:${eventsPort}`,
+  privateRegistryUrl: `http://localhost:${registryPort}/registry`,
   secretKeys: {
     identities: 'secret-identities',
-    events: 'secret-events'
+    events: 'secret-events',
+    registry: 'secret-registry-internal'
   }
 }

api/config/type/schema.json

@@ -12,8 +12,8 @@
   "additionalProperties": false,
   "required": [
     "cipherPassword",
-    "dataDir",
     "privateDirectoryUrl",
+    "privateRegistryUrl",
     "mongoUrl",
     "pluginCategories",
     "port",
@@ -25,7 +25,8 @@
       "type": "string"
     },
     "dataDir": {
-      "type": "string"
+      "type": ["string", "null"],
+      "description": "Optional. When set, the legacy plugins volume at <dataDir>/plugins is read by the v6.0 boot migration. Drops with v7.0."
     },
     "tmpDir": {
       "type": "string"
@@ -57,8 +58,14 @@
     "privateEventsUrl": {
       "type": "string"
     },
+    "privateRegistryUrl": {
+      "type": "string",
+      "pattern": "^https?://",
+      "description": "Internal URL used by the API to call registry server-to-server. The UI hits /registry on the same domain."
+    },
     "secretKeys": {
       "type": "object",
+      "required": ["registry"],
       "properties": {
         "limits": {
           "type": [
@@ -71,6 +78,10 @@
         },
         "identities": {
           "type": "string"
+        },
+        "registry": {
+          "type": "string",
+          "description": "x-secret-key shared with registry. Required for save-time validation, prepare-hook downloads, and the v6.0 first-boot migration."
         }
       }
     },
@@ -97,4 +108,4 @@
     "get": {},
     "has": {}
   }
-}
+}

api/package.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "@data-fair/lib-express": "^1.22.5",
     "@data-fair/lib-node": "^2.12.1",
+    "@data-fair/lib-node-registry": "^0.4.0",
     "@data-fair/lib-utils": "^1.10.1",
     "@data-fair/processings-shared": "*",
     "ajv": "^8.17.1",

api/src/admin/status.ts

@@ -8,11 +8,13 @@ const volumeStatus = async () => {
   await fs.writeFile(`${config.dataDir}/check-access.txt`, 'ok')
 }
 
-export const getStatus = async (req: Request) =>
-  runHealthChecks(req, [
-    { fn: mongoStatus, name: 'mongodb' },
-    { fn: volumeStatus, name: 'data volume' }
-  ])
+export const getStatus = async (req: Request) => {
+  const checks: Array<{ fn: (req: Request) => Promise<void>; name: string }> = [
+    { fn: mongoStatus, name: 'mongodb' }
+  ]
+  if (config.dataDir) checks.push({ fn: volumeStatus, name: 'data volume' })
+  return runHealthChecks(req, checks)
+}
 
 // Helper functions
 const getSingleStatus = async (req: Request, fn: (req: Request) => Promise<void>, name: string) => {

api/src/app.ts

@@ -3,8 +3,6 @@ import express from 'express'
 import { session, errorHandler, createSiteMiddleware, createSpaMiddleware, defaultNonceCSPDirectives } from '@data-fair/lib-express/index.js'
 import identitiesRouter from './misc/routers/identities.ts'
 import limitsRouter from './limits/router.ts'
-import pluginsRegistryRouter from './plugins-registry/router.ts'
-import pluginsRouter from './plugins/router.ts'
 import processingsRouter from './processings/router.ts'
 import runsRouter from './runs/router.ts'
 import adminRouter from './admin/router.ts'
@@ -28,8 +26,6 @@ app.get('/api/v1/_ping', (req, res) => {
 })
 
 app.use('/api/identities', identitiesRouter)
-app.use('/api/v1/plugins-registry', pluginsRegistryRouter)
-app.use('/api/v1/plugins', pluginsRouter)
 app.use('/api/v1/processings', processingsRouter)
 app.use('/api/v1/runs', runsRouter)
 app.use('/api/v1/limits', limitsRouter)

api/src/config.ts

@@ -1,13 +1,27 @@
+import path from 'node:path'
+import os from 'node:os'
 import type { ApiConfig } from '../config/type/index.ts'
 import { assertValid } from '../config/type/index.ts'
 import config from 'config'
 
 assertValid(config, { lang: 'en', name: 'config', internal: true })
 
-const apiConfig = config as unknown as ApiConfig
+const rawConfig = config as unknown as ApiConfig
+
+if (!rawConfig.tmpDir) {
+  rawConfig.tmpDir = rawConfig.dataDir
+    ? path.join(rawConfig.dataDir, 'tmp')
+    : path.join(os.tmpdir(), 'data-fair-processings')
+}
+
+const apiConfig = rawConfig as ApiConfig & { tmpDir: string }
+
+// cacheDir is always a subdirectory of tmpDir.
+export const registryCacheDir = path.join(apiConfig.tmpDir, 'registry-cache')
+
 export default apiConfig
 
 export const uiConfig = {
-  pluginCategories: apiConfig.pluginCategories,
+  pluginCategories: apiConfig.pluginCategories
 }
 export type UiConfig = typeof uiConfig

api/src/misc/routers/test-env.ts

@@ -51,6 +51,23 @@ router.get('/raw-processing/:id', async (req, res, next) => {
   }
 })
 
+// Patch a processing document without validation (used by tests to put a
+// processing into states that the normal API guards prevent, e.g. setting
+// plugin to a value that doesn't resolve in the registry).
+router.patch('/raw-processing/:id', async (req, res, next) => {
+  try {
+    const result = await mongo.processings.updateOne(
+      { _id: req.params.id },
+      { $set: req.body }
+    )
+    if (result.matchedCount === 0) return res.status(404).json({ error: 'processing not found' })
+    const processing = await mongo.processings.findOne({ _id: req.params.id })
+    res.json(processing)
+  } catch (err) {
+    next(err)
+  }
+})
+
 // Return the raw MongoDB document for a run
 router.get('/raw-run/:id', async (req, res, next) => {
   try {
@@ -93,11 +110,14 @@ router.post('/set-config', (req, res, next) => {
   }
 })
 
-// Wipe the installed plugins directory (used between test runs)
+// Wipe the installed plugins directory (used between test runs).
+// No-op when the legacy plugins volume isn't mounted (config.dataDir unset).
 router.delete('/plugins', async (req, res, next) => {
   try {
-    const pluginsDir = path.resolve(config.dataDir, 'plugins')
-    await fs.emptyDir(pluginsDir)
+    if (config.dataDir) {
+      const pluginsDir = path.resolve(config.dataDir, 'plugins')
+      await fs.emptyDir(pluginsDir)
+    }
     res.json({ ok: true })
   } catch (err) {
     next(err)