diff --git a/.gitignore b/.gitignore index e1bc6426..5aaaa484 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ copilot/ tools/ infra/.env +.DS_Store diff --git a/app/package-lock.json b/app/package-lock.json index ea70ea31..bb41ca00 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1636,6 +1636,10 @@ "resolved": "packages/language-config-optimization", "link": true }, + "node_modules/@mdeo/language-csv": { + "resolved": "packages/language-csv", + "link": true + }, "node_modules/@mdeo/language-expression": { "resolved": "packages/language-expression", "link": true @@ -1648,6 +1652,10 @@ "resolved": "packages/language-model", "link": true }, + "node_modules/@mdeo/language-model-csv": { + "resolved": "packages/language-model-csv", + "link": true + }, "node_modules/@mdeo/language-model-transformation": { "resolved": "packages/language-model-transformation", "link": true @@ -1700,6 +1708,10 @@ "resolved": "packages/service-config-optimization", "link": true }, + "node_modules/@mdeo/service-csv": { + "resolved": "packages/service-csv", + "link": true + }, "node_modules/@mdeo/service-metamodel": { "resolved": "packages/service-metamodel", "link": true @@ -1708,6 +1720,14 @@ "resolved": "packages/service-model", "link": true }, + "node_modules/@mdeo/service-model-common": { + "resolved": "packages/service-model-common", + "link": true + }, + "node_modules/@mdeo/service-model-csv": { + "resolved": "packages/service-model-csv", + "link": true + }, "node_modules/@mdeo/service-model-transformation": { "resolved": "packages/service-model-transformation", "link": true @@ -7860,6 +7880,18 @@ "vscode-languageserver-types": "^3.17.5" } }, + "packages/language-csv": { + "name": "@mdeo/language-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, "packages/language-expression": { "name": "@mdeo/language-expression", "version": "0.1.0", @@ -7915,6 +7947,20 @@ "vscode-languageserver-types": "^3.17.5" } }, + "packages/language-model-csv": { + "name": "@mdeo/language-model-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-metamodel": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, "packages/language-model-transformation": { "name": "@mdeo/language-model-transformation", "version": "0.1.0", @@ -8109,6 +8155,23 @@ "vite": "~7.3.1" } }, + "packages/service-csv": { + "name": "@mdeo/service-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "tsx": "^4.21.0", + "vite": "^7.3.1" + } + }, "packages/service-metamodel": { "name": "@mdeo/service-metamodel", "version": "0.1.0", @@ -8136,9 +8199,11 @@ "@mdeo/editor-model": "^0.1.0", "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", "lucide": "^0.575.0" }, "devDependencies": { @@ -8147,6 +8212,38 @@ "vite": "^7.3.1" } }, + "packages/service-model-common": { + "name": "@mdeo/service-model-common", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-model": "^0.1.0", + "@mdeo/service-common": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, + "packages/service-model-csv": { + "name": "@mdeo/service-model-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-model-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1", + "tsx": "~4.21.0", + "vite": "~7.3.1" + } + }, "packages/service-model-transformation": { "name": "@mdeo/service-model-transformation", "version": "0.1.0", diff --git a/app/packages/language-csv/package.json b/app/packages/language-csv/package.json new file mode 100644 index 00000000..25923f33 --- /dev/null +++ b/app/packages/language-csv/package.json @@ -0,0 +1,16 @@ +{ + "name": "@mdeo/language-csv", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist", "src"], + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/language-csv/src/csvPlugin.ts b/app/packages/language-csv/src/csvPlugin.ts new file mode 100644 index 00000000..9b18a254 --- /dev/null +++ b/app/packages/language-csv/src/csvPlugin.ts @@ -0,0 +1,39 @@ +import { + type LangiumLanguagePlugin, + type LangiumLanguagePluginProvider, + type LanguageServices, + type ExternalReferenceAdditionalServices +} from "@mdeo/language-common"; +import { + DefaultAstSerializer, + SerializerFormatter, + registerDefaultTokenSerializers, + DefaultExternalReferenceCollector +} from "@mdeo/language-shared"; +import type { ServerContributionPlugin } from "@mdeo/plugin"; +import { CsvFileRule, CsvTerminals } from "./grammar/csvRules.js"; + +function createCsvPlugin(): LangiumLanguagePlugin { + return { + rootRule: CsvFileRule, + additionalTerminals: CsvTerminals, + module: { + references: { + ExternalReferenceCollector: () => new DefaultExternalReferenceCollector() + }, + lsp: { + Formatter: (services: LanguageServices & ExternalReferenceAdditionalServices) => new SerializerFormatter(services) + }, + AstSerializer: (services: LanguageServices & ExternalReferenceAdditionalServices) => new DefaultAstSerializer(services) + }, + postCreate(services: LanguageServices & ExternalReferenceAdditionalServices) { + registerDefaultTokenSerializers(services); + } + }; +} + +export const csvPluginProvider: LangiumLanguagePluginProvider = { + create(_contributionPlugins: ServerContributionPlugin[], _languageJsUrl?: string): LangiumLanguagePlugin { + return createCsvPlugin(); + } +}; diff --git a/app/packages/language-csv/src/grammar/csvRules.ts b/app/packages/language-csv/src/grammar/csvRules.ts new file mode 100644 index 00000000..c78cb92d --- /dev/null +++ b/app/packages/language-csv/src/grammar/csvRules.ts @@ -0,0 +1,19 @@ +import { createTerminal, createRule, optional, WS } from "@mdeo/language-common"; +import { CsvFile } from "./csvTypes.js"; + +/** + * Matches a whole CSV file as one token. The CSV language does no structural + * parsing of its own, the content is interpreted where it is imported. + * + * The pattern requires at least one character, since a terminal that can match + * the empty string would never advance the lexer. An empty file is handled by + * making the assignment optional in the rule below rather than by allowing an + * empty match here. + */ +export const ANY_TEXT = createTerminal("ANY_TEXT").as(/[\s\S]+/); + +export const CsvFileRule = createRule("CsvFileRule") + .returns(CsvFile) + .as(({ set }) => [optional(set("content", ANY_TEXT))]); + +export const CsvTerminals = [WS, ANY_TEXT]; diff --git a/app/packages/language-csv/src/grammar/csvTypes.ts b/app/packages/language-csv/src/grammar/csvTypes.ts new file mode 100644 index 00000000..5596072c --- /dev/null +++ b/app/packages/language-csv/src/grammar/csvTypes.ts @@ -0,0 +1,15 @@ +import { createInterface, Optional, type ASTType } from "@mdeo/language-common"; + +/** + * A CSV file's contents, held as a single blob of text. + * + * `content` is optional because an empty file is a valid CSV file. The + * workbench offers a "Create New CSV" action, which produces an empty file, so + * requiring content would put every newly created CSV straight into a parse + * error state. + */ +export const CsvFile = createInterface("CsvFile").attrs({ + content: Optional(String) +}); + +export type CsvFileType = ASTType; diff --git a/app/packages/language-csv/src/index.ts b/app/packages/language-csv/src/index.ts new file mode 100644 index 00000000..2b1a4096 --- /dev/null +++ b/app/packages/language-csv/src/index.ts @@ -0,0 +1,3 @@ +export * from "./grammar/csvTypes.js"; +export * from "./grammar/csvRules.js"; +export * from "./csvPlugin.js"; diff --git a/app/packages/language-csv/tsconfig.json b/app/packages/language-csv/tsconfig.json new file mode 100644 index 00000000..b11a3e26 --- /dev/null +++ b/app/packages/language-csv/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-common" }, + { "path": "../language-shared" } + ] +} diff --git a/app/packages/language-model-csv/package.json b/app/packages/language-model-csv/package.json new file mode 100644 index 00000000..40044e22 --- /dev/null +++ b/app/packages/language-model-csv/package.json @@ -0,0 +1,18 @@ +{ + "name": "@mdeo/language-model-csv", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist", "src"], + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-metamodel": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/language-model-csv/src/csvPlugin.ts b/app/packages/language-model-csv/src/csvPlugin.ts new file mode 100644 index 00000000..541d0d5d --- /dev/null +++ b/app/packages/language-model-csv/src/csvPlugin.ts @@ -0,0 +1,90 @@ +import { + type LangiumLanguagePlugin, + type LangiumLanguagePluginProvider, + type ExternalReferenceAdditionalServices, + GrammarDeserializationContext, + ID, + NEWLINE, + HIDDEN_NEWLINE, + INT, + FLOAT, + STRING +} from "@mdeo/language-common"; +import { + DefaultAstSerializer, + IdValueConverter, + SerializerFormatter, + registerDefaultTokenSerializers, + NewlineAwareTokenBuilder, + addExternalReferenceCollectionPhase, + ActionHandlerRegistry, + type ActionHandlerRegistryAdditionalServices, + DefaultActionProvider +} from "@mdeo/language-shared"; +import { generateModelContributionGrammar, ModelTerminals, ModelScopeProvider, ModelExternalReferenceCollector } from "@mdeo/language-model"; +import { Class } from "@mdeo/language-metamodel"; +import { createModelCsvContributionPlugin } from "./plugin/modelCsvContributionPlugin.js"; + +/** + * Combined services type for the standalone model-csv language. + * Reuses the base Model language's services, since the standalone grammar's + * root is a real `Model` node (an `import` line plus the CSV import block). + */ +export type ModelCsvServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; + +/** + * Deserialization context for the CSV grammar. + * Provides the external interface types and common terminals so the CSV + * grammar can be deserialized from its serialized form. + */ +const csvDeserializationContext = GrammarDeserializationContext.create([Class], [], [ID, NEWLINE, HIDDEN_NEWLINE, INT, FLOAT, STRING]); + +/** + * The root rule for the standalone model-csv language. + * Generated from the CSV contribution plugin's grammar. + */ +const csvRootRule = generateModelContributionGrammar(createModelCsvContributionPlugin(), csvDeserializationContext); + +/** + * The plugin for the standalone model-csv generated language. + * Its root type is a real `Model` node, so it reuses the base Model language's + * own scope provider and external reference collector directly. + */ +const modelCsvPlugin: LangiumLanguagePlugin = { + rootRule: csvRootRule, + additionalTerminals: ModelTerminals, + module: { + parser: { + TokenBuilder: () => new NewlineAwareTokenBuilder(new Set(["{"]), new Set(["("]), new Set(["}", ")"])), + ValueConverter: () => new IdValueConverter(), + ParserConfig: () => ({ + maxLookahead: 4 + }) + }, + references: { + ScopeProvider: (services) => new ModelScopeProvider(services), + ExternalReferenceCollector: () => new ModelExternalReferenceCollector() + }, + lsp: { + Formatter: (services) => new SerializerFormatter(services) + }, + AstSerializer: (services) => new DefaultAstSerializer(services), + action: { + ActionHandlerRegistry: () => new ActionHandlerRegistry(), + ActionProvider: () => new DefaultActionProvider() + } + }, + postCreate(services) { + registerDefaultTokenSerializers(services); + addExternalReferenceCollectionPhase(services); + } +}; + +/** + * Provider for the standalone model-csv language plugin. + */ +export const modelCsvPluginProvider: LangiumLanguagePluginProvider = { + create(): LangiumLanguagePlugin { + return modelCsvPlugin; + } +}; diff --git a/app/packages/language-model-csv/src/features/csvImport.ts b/app/packages/language-model-csv/src/features/csvImport.ts new file mode 100644 index 00000000..3607655a --- /dev/null +++ b/app/packages/language-model-csv/src/features/csvImport.ts @@ -0,0 +1,248 @@ +import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "@mdeo/language-model"; +import { parseCsv } from "@mdeo/language-shared"; + +export interface CsvColumnMappingEntry { + csvColumn: string; + property: string; +} + +export interface CsvImportEntry { + className: string; + csvText: string; + /** + * Explicit CSV column to property mappings. When omitted or empty, columns + * are matched to properties by name (the default behavior). + */ + mappings?: CsvColumnMappingEntry[]; +} + +export interface CsvImportResult { + instances: ModelDataInstance[]; + links: ModelData["links"]; + warnings: string[]; +} + +/** + * Maps one or more CSV files onto a metamodel, producing model instances and links. + * Each entry maps one class to one CSV file (one row = one instance). + * Column names must match property names in the metamodel class. + * Cross-object references use a reserved `_id` column as the row identifier, + * with reference columns holding the target `_id` value. + */ +export function importCsvEntries( + entries: CsvImportEntry[], + metamodelClasses: MetamodelClassInfo[] +): CsvImportResult { + const warnings: string[] = []; + const allInstances: ModelDataInstance[] = []; + const allLinks: ModelData["links"] = []; + + const idMap = new Map(); + + /** + * Row numbering restarts at zero for every entry, but two entries can import + * the same class from different files, which would then generate the same + * instance names twice. Each entry is numbered from the count of rows already + * imported for its class, so names stay unique across the whole model. Links + * refer to instances by name, so a collision would silently point at the + * wrong object. + */ + const nameOffsets = new Map(); + const rowsPerClass = new Map(); + + for (const [entryIndex, entry] of entries.entries()) { + const classInfo = metamodelClasses.find(c => c.name === entry.className); + if (!classInfo) { + warnings.push(`Class '${entry.className}' not found in metamodel — skipping.`); + continue; + } + + const rows = parseCsv(entry.csvText); + if (rows.length < 2) { + warnings.push(`CSV for class '${entry.className}' has no data rows — skipping.`); + continue; + } + + const header = rows[0]; + const dataRows = rows.slice(1); + const idColIndex = header.indexOf("_id"); + + const columnToProperty = resolveColumnMapping(entry, header, classInfo, warnings); + + const nameOffset = rowsPerClass.get(entry.className) ?? 0; + nameOffsets.set(entryIndex, nameOffset); + rowsPerClass.set(entry.className, nameOffset + dataRows.length); + + dataRows.forEach((row, rowIndex) => { + const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); + const instanceName = `${entry.className}_${nameOffset + rowIndex}`; + + if (idColIndex >= 0) { + const idValue = normalizedRow[idColIndex]; + if (idValue) { + idMap.set(`${entry.className}:${idValue}`, instanceName); + } + } + + const properties: Record = {}; + header.forEach((colName, colIndex) => { + if (colName === "_id") return; + const propName = columnToProperty.get(colName); + if (!propName) return; + const prop = classInfo.properties.find(p => p.name === propName); + if (!prop) return; + properties[propName] = convertCellValue(normalizedRow[colIndex], prop); + }); + + allInstances.push({ + name: instanceName, + className: entry.className, + properties + }); + }); + } + + for (const [entryIndex, entry] of entries.entries()) { + const classInfo = metamodelClasses.find(c => c.name === entry.className); + if (!classInfo) continue; + + const rows = parseCsv(entry.csvText); + if (rows.length < 2) continue; + + const header = rows[0]; + const dataRows = rows.slice(1); + const columnToProperty = resolveColumnMapping(entry, header, classInfo, []); + + const refCols = header.filter(colName => { + const propName = columnToProperty.get(colName); + const prop = propName ? classInfo.properties.find(p => p.name === propName) : undefined; + return prop?.isReference; + }); + + if (refCols.length === 0) continue; + + const nameOffset = nameOffsets.get(entryIndex) ?? 0; + + dataRows.forEach((row, rowIndex) => { + const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); + const sourceInstanceName = `${entry.className}_${nameOffset + rowIndex}`; + + refCols.forEach(colName => { + const propName = columnToProperty.get(colName)!; + const prop = classInfo.properties.find(p => p.name === propName)!; + const rawValue = normalizedRow[header.indexOf(colName)]; + if (!rawValue) return; + + const targetIds = rawValue.split(";").map(s => s.trim()).filter(Boolean); + targetIds.forEach(targetId => { + const targetInstanceName = idMap.get(`${prop.referencedClass}:${targetId}`); + if (!targetInstanceName) { + warnings.push(`Reference '${targetId}' in column '${colName}' of '${entry.className}' row ${rowIndex + 2} could not be resolved.`); + return; + } + allLinks.push({ + sourceName: sourceInstanceName, + sourceProperty: propName, + targetName: targetInstanceName, + targetProperty: null + }); + }); + }); + }); + } + + return { instances: allInstances, links: allLinks, warnings }; +} + +/** + * Resolves which CSV column maps to which metamodel property for one entry. + * + * When the entry has an explicit mapping, only the mapped columns are included + * (this is also how a column can be intentionally skipped). Otherwise, columns + * are matched to properties by name, and unmatched columns are warned about. + * + * @param entry The import entry, whose optional `mappings` take precedence + * @param header The CSV header row + * @param classInfo The metamodel class the CSV is imported into + * @param warnings Warnings accumulator; mutated in place + * @returns A map from CSV column name to metamodel property name + */ +function resolveColumnMapping( + entry: CsvImportEntry, + header: string[], + classInfo: MetamodelClassInfo, + warnings: string[] +): Map { + if (entry.mappings != undefined && entry.mappings.length > 0) { + const columnToProperty = new Map(); + for (const mapping of entry.mappings) { + if (!header.includes(mapping.csvColumn)) { + warnings.push(`CSV for '${entry.className}' has no column '${mapping.csvColumn}' referenced by its mapping — skipping.`); + continue; + } + if (!classInfo.properties.some(p => p.name === mapping.property)) { + warnings.push(`Class '${entry.className}' has no property '${mapping.property}' referenced by its mapping — skipping.`); + continue; + } + columnToProperty.set(mapping.csvColumn, mapping.property); + } + return columnToProperty; + } + + const propertyNames = new Set(classInfo.properties.map(p => p.name)); + const columnToProperty = new Map( + header.filter(h => h !== "_id" && propertyNames.has(h)).map(h => [h, h]) + ); + const unknownCols = header.filter(h => h !== "_id" && !columnToProperty.has(h)); + if (unknownCols.length > 0) { + warnings.push(`CSV for '${entry.className}' has unknown columns: ${unknownCols.join(", ")} — they will be ignored.`); + } + return columnToProperty; +} + +export interface MetamodelPropertyInfo { + name: string; + type: "string" | "int" | "long" | "double" | "float" | "boolean" | "enum" | "reference"; + enumEntries?: string[]; + isReference?: boolean; + referencedClass?: string; +} + +export interface MetamodelClassInfo { + name: string; + properties: MetamodelPropertyInfo[]; +} + +function convertCellValue(rawValue: string, prop: MetamodelPropertyInfo): ModelDataPropertyValue { + if (!rawValue || rawValue.trim() === "") return null; + const v = rawValue.trim(); + switch (prop.type) { + case "int": + case "long": { + const n = parseInt(v, 10); + return isNaN(n) ? v : n; + } + case "double": + case "float": { + const n = parseFloat(v); + return isNaN(n) ? v : n; + } + case "boolean": + return v.toLowerCase() === "true"; + case "enum": + return { enum: v }; + default: + return v; + } +} + +function normalizeRow(row: string[], expectedLength: number, rowNumber: number, warnings: string[]): string[] { + if (row.length === expectedLength) return row; + if (row.length < expectedLength) { + warnings.push(`Row ${rowNumber} has fewer columns than the header; missing values treated as blank.`); + return [...row, ...Array(expectedLength - row.length).fill("")]; + } + warnings.push(`Row ${rowNumber} has more columns than the header; extra values ignored.`); + return row.slice(0, expectedLength); +} + diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts new file mode 100644 index 00000000..c474c807 --- /dev/null +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -0,0 +1,44 @@ +import { createRule, many, optional, or, ref, group, createExternalTerminalRule } from "@mdeo/language-common"; +import { CsvClassImport, CsvColumnMapping, CsvImportBlock, ExternalClass } from "./csvImportTypes.js"; + +/** + * Stand-ins for the base language's common terminals, so this grammar's + * serialized form marks them as external rather than inlining duplicate + * definitions. The real terminals are supplied via the deserialization + * context wherever this grammar is merged in. + */ +const ID = createExternalTerminalRule("ID"); +const STRING = createExternalTerminalRule("STRING"); +const NEWLINE = createExternalTerminalRule("NEWLINE"); + +export const CsvColumnMappingRule = createRule("CsvColumnMappingRule") + .returns(CsvColumnMapping) + .as(({ set }) => [set("csvColumn", STRING), "=", set("property", ID)]); + +/** + * A class import, with an optional explicit column mapping block. + * + * The mapping block uses plain nested braces, like the nested blocks in the + * other DSLs. Nothing needs to separate it from `file`: the block can only + * start with "{", while the enclosing import list continues with an ID or ends + * with "}", so the parser can always tell which one it is from a single token. + */ +export const CsvClassImportRule = createRule("CsvClassImportRule") + .returns(CsvClassImport) + .as(({ set, add }) => [ + set("class", ref(ExternalClass, ID)), + "from", + set("file", STRING), + optional( + group("{", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "}") + ) + ]); + +/** + * The content of a CSV import block (everything between the braces). + * The `import CSV` keywords themselves are added by the wrapper rule the + * Model language builds around this contribution, not by this rule. + */ +export const CsvImportContentRule = createRule("CsvImportContentRule") + .returns(CsvImportBlock) + .as(({ add }) => ["{", many(or(add("imports", CsvClassImportRule), NEWLINE)), "}"]); diff --git a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts new file mode 100644 index 00000000..03bcc104 --- /dev/null +++ b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts @@ -0,0 +1,53 @@ +import { createInterface, createExternalInterface, Ref, type ASTType } from "@mdeo/language-common"; +import type { ClassType } from "@mdeo/language-metamodel"; + +/** + * Stand-in for the metamodel's Class interface, used only so this grammar's + * serialized form marks cross-references to it as external rather than + * inlining a duplicate "Class" interface definition. The real Class type is + * supplied via the deserialization context wherever this grammar is merged in. + */ +export const ExternalClass = createExternalInterface("Class"); + +/** + * An explicit CSV column to metamodel property mapping. + * Format: "csvColumnName" = modelPropertyName + */ +export const CsvColumnMapping = createInterface("CsvColumnMapping").attrs({ + csvColumn: String, + property: String +}); + +export type CsvColumnMappingType = ASTType; + +/** + * A single CSV class import entry. + * Maps a metamodel class to a CSV file, with an optional explicit column mapping. + * Format: + * ClassName from "path/to/file.csv" + * or, with an explicit mapping: + * ClassName from "path/to/file.csv" { + * "csvColumnName" = modelPropertyName + * } + * When no mapping list is given, columns are matched to properties by name. + */ +export const CsvClassImport = createInterface("CsvClassImport").attrs({ + class: Ref(() => ExternalClass), + file: String, + mappings: [CsvColumnMapping] +}); + +export type CsvClassImportType = ASTType; + +/** + * The content of a CSV import block (everything between the braces). + * Format: + * { + * ClassName from "file.csv" + * } + */ +export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ + imports: [CsvClassImport] +}); + +export type CsvImportBlockType = ASTType; diff --git a/app/packages/language-model-csv/src/index.ts b/app/packages/language-model-csv/src/index.ts new file mode 100644 index 00000000..a79e548c --- /dev/null +++ b/app/packages/language-model-csv/src/index.ts @@ -0,0 +1,5 @@ +export * from "./grammar/csvImportTypes.js"; +export * from "./grammar/csvImportRules.js"; +export * from "./plugin/modelCsvContributionPlugin.js"; +export * from "./features/csvImport.js"; +export * from "./csvPlugin.js"; diff --git a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts new file mode 100644 index 00000000..16704c3b --- /dev/null +++ b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts @@ -0,0 +1,57 @@ +import { ModelContributionPlugin } from "@mdeo/language-model"; +import { GrammarSerializer, type SerializedGrammar } from "@mdeo/language-common"; +import { CsvClassImportRule, CsvImportContentRule } from "../grammar/csvImportRules.js"; +import { CsvImportBlock } from "../grammar/csvImportTypes.js"; + +/** + * The unique name for the CSV contribution plugin. + */ +export const CSV_PLUGIN_NAME = "csv"; + +/** + * The unique plugin ID for the CSV contribution plugin. + */ +export const CSV_PLUGIN_ID = "model-csv"; + +/** + * The language key used to retrieve services for the standalone model-csv language. + */ +export const MODEL_CSV_LANGUAGE_KEY = "model-csv"; + +/** + * Creates the serialized grammar for the CSV import rules. + * + * @returns The serialized grammar + */ +function createCsvGrammar(): SerializedGrammar { + const serializer = new GrammarSerializer({ + rules: [CsvClassImportRule, CsvImportContentRule], + additionalTerminals: [] + }); + return serializer.grammar; +} + +/** + * Creates the CSV contribution plugin. + * This plugin provides the `import CSV { ClassName from "file.csv" }` syntax for the model language. + * + * @returns The ModelContributionPlugin for CSV + */ +export function createModelCsvContributionPlugin(): ModelContributionPlugin { + return { + id: CSV_PLUGIN_ID, + type: ModelContributionPlugin.TYPE, + name: CSV_PLUGIN_NAME, + languageKey: MODEL_CSV_LANGUAGE_KEY, + grammar: createCsvGrammar(), + imports: [ + { + name: "CSV", + ruleName: CsvImportContentRule.name, + interfaceName: CsvImportBlock.name + } + ], + dependencies: [], + exportedTypes: [] + }; +} diff --git a/app/packages/language-model-csv/tsconfig.json b/app/packages/language-model-csv/tsconfig.json new file mode 100644 index 00000000..a2d1512f --- /dev/null +++ b/app/packages/language-model-csv/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-common" }, + { "path": "../language-model" }, + { "path": "../language-metamodel" }, + { "path": "../language-shared" } + ] +} diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 3c057384..59d61d62 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -6,12 +6,14 @@ import { GHorizontalDivider, EdgeLayoutMetadataUtil, NodeLayoutMetadataUtil, - parseIdentifier + parseIdentifier, + resolveRelativePath, + parseCsv } from "@mdeo/language-shared"; import type { ModelIdRegistry, GraphMetadata } from "@mdeo/language-shared"; import type { NodeLayoutMetadata, EdgeLayoutMetadata } from "@mdeo/protocol-common"; import { ID } from "@mdeo/language-common"; -import { resolveClassChain, type ClassType } from "@mdeo/language-metamodel"; +import { resolveClassChain, type ClassType, type PropertyType } from "@mdeo/language-metamodel"; import type { PartialModel, PartialObjectInstance, @@ -33,12 +35,27 @@ import { type EnumValueType, type SingleValueType } from "../../grammar/modelTypes.js"; +import { getWrapperInterfaceName } from "../../plugin/resolvePlugins.js"; const { injectable } = sharedImport("inversify"); const { GGraph } = sharedImport("@eclipse-glsp/server"); type GGraphType = ReturnType["proxy"]; +/** + * Reserved CSV column holding a row's identifier for cross-object references. + * It is not a metamodel property, so it is never displayed as one. + */ +const CSV_ID_COLUMN = "_id"; + +/** + * A CSV column that maps onto a metamodel property of the imported class. + */ +interface CsvColumnBinding { + index: number; + property: PropertyType; +} + /** * Factory for creating GLSP graph models from model AST. * Transforms the model source into a visual graph representation @@ -56,9 +73,10 @@ export class ModelGModelFactory extends BaseGModelFactory { override async createModelInternal(sourceModel: PartialModel, idRegistry: ModelIdRegistry): Promise { const graph = GGraph.builder().id("model-graph").addCssClass("editor-model").build(); - const extracted = this.extractElements(sourceModel); + const extracted = await this.extractElements(sourceModel); await this.createObjectNodes(graph, extracted.objects, idRegistry); await this.createLinkEdges(graph, extracted.links, idRegistry); + await this.createCsvNodes(graph, sourceModel); return graph; } @@ -68,10 +86,10 @@ export class ModelGModelFactory extends BaseGModelFactory { * @param model The model containing elements * @returns An object containing separate arrays of objects and links */ - private extractElements(model: PartialModel): { + private async extractElements(model: PartialModel): Promise<{ objects: PartialObjectInstance[]; links: PartialLink[]; - } { + }> { const objects: PartialObjectInstance[] = []; const links: PartialLink[] = []; @@ -469,4 +487,222 @@ export class ModelGModelFactory extends BaseGModelFactory { return nodes; } + /** + * Renders diagram nodes for the CSV import contribution, if present. + * + * The Model language's grammar has no compile-time knowledge of CSV (it's a + * plugin contribution resolved generically via `model.imports`), but rendering + * synthetic instance nodes from CSV row data has no equivalent generic extension + * point today, so this reads the CSV plugin's contributed import structurally + * by its wrapper type name rather than importing its types directly (which would + * create a circular package dependency, since language-model-csv depends on + * language-model for the contribution plugin contract). + */ + private async createCsvNodes(graph: GGraphType, model: PartialModel): Promise { + interface CsvClassImportShape { + class?: { ref?: ClassType }; + file?: string; + mappings?: { csvColumn?: string; property?: string }[]; + } + + const csvImport = model.imports?.find((imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV")); + const csvImportContent = (csvImport as { content?: { imports?: CsvClassImportShape[] } } | undefined)?.content; + if (csvImportContent?.imports == undefined) return; + const doc = this.modelState.sourceModel?.$document; + if (!doc) return; + + const validatedMetadata = await this.modelState.getValidatedMetadata(); + + let nodeIndex = 0; + // Numbered per class rather than per import, so two imports of the same + // class from different files do not produce the same instance names. + // Matches how the import itself names them. + const rowsPerClass = new Map(); + for (const entry of csvImportContent.imports) { + const classRef = entry.class?.ref as ClassType | undefined; + if (classRef == undefined) continue; + try { + const uri = resolveRelativePath(doc, entry.file ?? ""); + const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); + const rows = parseCsv(csvContent); + if (rows.length < 2) continue; + const classChain = resolveClassChain(classRef, this.reflection); + const classHierarchy = classChain.map((c) => c.name); + const columns = this.resolveCsvColumns(rows[0], classChain, entry.mappings); + const nameOffset = rowsPerClass.get(classRef.name) ?? 0; + rowsPerClass.set(classRef.name, nameOffset + rows.length - 1); + rows.slice(1).forEach((row: string[], rowIndex: number) => { + const instanceName = `${classRef.name}_${nameOffset + rowIndex}`; + const nodeId = `csv-node-${nodeIndex++}`; + const metadata = validatedMetadata.nodes[nodeId]?.meta ?? {}; + const node = GObjectNode.builder() + .id(nodeId) + .name(instanceName) + .typeName(classRef.name) + .classHierarchy(classHierarchy) + .meta(metadata) + .build(); + node.children.push( + ...this.createObjectHeader(nodeId, instanceName, classRef.name), + ...this.createCsvPropertyAssignments(nodeId, columns, row) + ); + graph.children.push(node); + }); + } catch { + // skip unreadable CSV + } + } + } + + /** + * Resolves which CSV columns correspond to properties of the imported class. + * + * When the import declares an explicit mapping, only the mapped columns are + * used, so a column can also be left out deliberately. Otherwise columns are + * matched to properties by name across the class' whole extension chain. Either + * way this mirrors how the import itself resolves them, so the diagram shows + * the same columns the import reads. Unmatched columns and the reserved `_id` + * row identifier are dropped, so they are simply not shown. + * + * @param header The CSV header row + * @param classChain The imported class and its extended classes + * @param mappings The import's explicit column to property mappings, if any + * @returns The column index and property for every column that maps to one + */ + private resolveCsvColumns( + header: string[], + classChain: ClassType[], + mappings?: { csvColumn?: string; property?: string }[] + ): CsvColumnBinding[] { + const propertiesByName = new Map(); + for (const cls of classChain) { + for (const property of cls.properties ?? []) { + if (property?.name != undefined && !propertiesByName.has(property.name)) { + propertiesByName.set(property.name, property); + } + } + } + + const columns: CsvColumnBinding[] = []; + + if (mappings != undefined && mappings.length > 0) { + for (const mapping of mappings) { + if (mapping.csvColumn == undefined || mapping.property == undefined) { + continue; + } + const index = header.indexOf(mapping.csvColumn); + const property = propertiesByName.get(mapping.property); + if (index >= 0 && property != undefined) { + columns.push({ index, property }); + } + } + return columns; + } + + header.forEach((columnName, index) => { + if (columnName === CSV_ID_COLUMN) { + return; + } + const property = propertiesByName.get(columnName); + if (property != undefined) { + columns.push({ index, property }); + } + }); + return columns; + } + + /** + * Creates the property assignments compartment for a CSV-imported node. + * + * Mirrors {@link createPropertyAssignments}, but reads values straight from a + * CSV row instead of from property assignment AST nodes, since CSV-imported + * instances have no backing AST. The labels are read-only because editing them + * could not be written back to the CSV file. + * + * @param nodeId The ID of the object node + * @param columns The columns that map to properties, from {@link resolveCsvColumns} + * @param row The CSV data row for this node + * @returns Array of GModelElements for the properties compartment + */ + private createCsvPropertyAssignments( + nodeId: string, + columns: CsvColumnBinding[], + row: string[] + ): GModelElement[] { + const propertyLabels: GPropertyLabel[] = []; + for (const { index, property } of columns) { + const rawValue = row[index]; + if (rawValue == undefined || rawValue.trim() === "") { + continue; + } + const propertyName = this.modelState.languageServices.AstSerializer.serializePrimitive( + { value: property.name }, + ID + ); + propertyLabels.push( + GPropertyLabel.builder() + .id(`${nodeId}__prop-${index}`) + .text(`${propertyName} = ${this.formatCsvValue(rawValue, property)}`) + .readonly(true) + .build() + ); + } + + if (propertyLabels.length === 0) { + return []; + } + + const divider = GHorizontalDivider.builder().type(ModelElementType.DIVIDER).id(`${nodeId}__divider`).build(); + const propertiesCompartment = GCompartment.builder() + .type(ModelElementType.COMPARTMENT) + .id(`${nodeId}__properties-compartment`) + .build(); + propertiesCompartment.children.push(...propertyLabels); + + return [divider, propertiesCompartment]; + } + + /** + * Formats a raw CSV cell for display, following the property's declared type. + * + * Uses the same type interpretation as the CSV import itself, so the diagram + * shows the value the import would produce, and renders it the same way + * hand-authored values are rendered. Values that don't parse as their declared + * numeric type fall back to being shown as text rather than as `NaN`. + * + * @param rawValue The raw cell text + * @param property The metamodel property the column maps to + * @returns The formatted value string + */ + private formatCsvValue(rawValue: string, property: PropertyType): string { + const value = rawValue.trim(); + const propertyType = property.type as + | { name?: string; enum?: { ref?: { name?: string }; $refText?: string } } + | undefined; + + const enumName = + (propertyType?.enum?.ref as { name?: string } | undefined)?.name ?? + (propertyType?.enum != undefined ? parseIdentifier(propertyType.enum.$refText ?? "?") : undefined); + if (enumName != undefined) { + const serializer = this.modelState.languageServices.AstSerializer; + return `${serializer.serializePrimitive({ value: enumName }, ID)}.${serializer.serializePrimitive({ value }, ID)}`; + } + + switch (propertyType?.name) { + case "int": + case "long": { + const parsed = parseInt(value, 10); + return isNaN(parsed) ? `"${value}"` : String(parsed); + } + case "double": + case "float": { + const parsed = parseFloat(value); + return isNaN(parsed) ? `"${value}"` : String(parsed); + } + case "boolean": + return String(value.toLowerCase() === "true"); + default: + return `"${value}"`; + } + } } diff --git a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts index b83bd46d..7019694a 100644 --- a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts +++ b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts @@ -13,6 +13,7 @@ import { } from "@mdeo/language-shared"; import { ModelElementType } from "@mdeo/protocol-model"; import type { PartialModel, PartialObjectInstance, PartialLink } from "../../grammar/modelPartialTypes.js"; +import { getWrapperInterfaceName } from "../../plugin/resolvePlugins.js"; const { injectable, inject } = sharedImport("inversify"); @@ -63,10 +64,49 @@ export class ModelMetadataManager extends MetadataManager { this.extractObjectMetadata(objects, idRegistry, nodes); this.extractLinkMetadata(links, idRegistry, edges, nodes); + this.extractCsvNodeMetadata(sourceModel, nodes); return { nodes, edges }; } + /** + * The metadata being validated by the current {@link validateMetadata} call. + * + * CSV node metadata has to be carried forward from the incoming metadata + * rather than derived from the model, but `extractGraphMetadata` only + * receives the source model, so this passes it down. It is scoped to a + * single call: it is restored in a `finally` block, so a nested or + * concurrent validation cannot see another document's metadata. + */ + private storedCurrentMetadata: GraphMetadata | undefined; + + override async validateMetadata( + sourceModel: PartialModel, + currentMetadata: GraphMetadata, + lastValidMetadata: GraphMetadata + ): Promise { + const previousMetadata = this.storedCurrentMetadata; + this.storedCurrentMetadata = currentMetadata; + try { + return await super.validateMetadata(sourceModel, currentMetadata, lastValidMetadata); + } finally { + this.storedCurrentMetadata = previousMetadata; + } + } + + private extractCsvNodeMetadata(sourceModel: PartialModel, nodes: Record): void { + const hasCsvImport = sourceModel.imports?.some( + (imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV") + ); + if (!hasCsvImport) return; + if (this.storedCurrentMetadata == undefined) return; + for (const [id, meta] of Object.entries(this.storedCurrentMetadata.nodes)) { + if (id.startsWith("csv-node-") && !nodes[id]) { + nodes[id] = meta; + } + } + } + /** * Extracts objects and links from the model. * diff --git a/app/packages/language-model/src/features/modelScopeProvider.ts b/app/packages/language-model/src/features/modelScopeProvider.ts index eb5d3acd..c8c634b4 100644 --- a/app/packages/language-model/src/features/modelScopeProvider.ts +++ b/app/packages/language-model/src/features/modelScopeProvider.ts @@ -92,7 +92,7 @@ export class ModelScopeProvider extends DefaultScopeProvider { override getScope(context: ReferenceInfo): Scope { const document = AstUtils.getDocument(context.container); - if (context.property === "class" && this.astReflection.isInstance(context.container, ObjectInstance)) { + if (context.property === "class") { return this.getObjectClassScope(context, document); } if (context.property === "name" && this.astReflection.isInstance(context.container, PropertyAssignment)) { @@ -115,16 +115,20 @@ export class ModelScopeProvider extends DefaultScopeProvider { } /** - * Gets the scope for object class references. - * Resolves the imported metamodel file and returns a scope with all accessible classes. + * Gets the scope for class references, wherever they occur in the document + * (an object instance, or a class reference inside a plugin-contributed import + * such as CSV, at any nesting depth). Resolves the imported metamodel file and + * returns a scope with all accessible classes. * * @param context The reference context * @param document The current document * @returns A scope containing all accessible classes from the imported metamodel */ private getObjectClassScope(context: ReferenceInfo, document: any): Scope { - const model = context.container.$container as ModelType; - const metamodelImport = model.import; + const model = AstUtils.getContainerOfType(context.container, (node): node is ModelType => + this.astReflection.isInstance(node, Model) + ); + const metamodelImport = model?.import; const relativePath = metamodelImport?.file; if (relativePath == undefined) { diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index cce0b9c9..6f06a9a2 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -2,6 +2,8 @@ import { createRule, or, many, + optional, + group, ref, ID, STRING, @@ -12,8 +14,8 @@ import { ML_COMMENT, SL_COMMENT, HIDDEN_NEWLINE, - optional, - group + type ParserRule, + type RuleEntry } from "@mdeo/language-common"; import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import { @@ -30,44 +32,24 @@ import { Link } from "./modelTypes.js"; -/** - * Boolean literal rule. - * Matches "true" or "false". - */ export const BOOLEAN = createRule("BOOLEAN") .returns(Boolean) .as(() => [or("true", "false")]); -/** - * Simple value rule. - * Matches string, number, or boolean literals. - */ export const SimpleValueRule = createRule("SimpleValueRule") .returns(SimpleValue) .as(({ set }) => [ or(set("stringValue", STRING), set("numberValue", FLOAT), set("numberValue", INT), set("booleanValue", BOOLEAN)) ]); -/** - * Enum value rule. - * Matches an enum entry using EnumName.Entry syntax. - */ export const EnumValueRule = createRule("EnumValueRule") .returns(EnumValue) .as(({ set }) => [set("enumRef", ref(Enum, ID)), ".", set("value", ref(EnumEntry, ID))]); -/** - * Single value rule. - * Matches either a simple value or an enum value. - */ export const SingleValueRule = createRule("SingleValueRule") .returns(SingleValue) .as(() => [or(SimpleValueRule, EnumValueRule)]); -/** - * List value rule. - * Matches values in square brackets with comma separation. - */ export const ListValueRule = createRule("ListValueRule") .returns(ListValue) .as(({ add }) => [ @@ -76,26 +58,14 @@ export const ListValueRule = createRule("ListValueRule") "]" ]); -/** - * Literal value rule. - * Matches any value type: simple, enum, or list. - */ export const LiteralValueRule = createRule("LiteralValueRule") .returns(LiteralValue) .as(() => [or(ListValueRule, SimpleValueRule, EnumValueRule)]); -/** - * Property assignment rule. - * Matches property assignments like "a = 100". - */ export const PropertyAssignmentRule = createRule("PropertyAssignmentRule") .returns(PropertyAssignment) .as(({ set }) => [set("name", ref(Property, ID)), "=", set("value", LiteralValueRule)]); -/** - * Object instance rule. - * Matches object definitions like "test : ClassName { ... }". - */ export const ObjectInstanceRule = createRule("ObjectInstanceRule") .returns(ObjectInstance) .as(({ set, add }) => [ @@ -107,10 +77,6 @@ export const ObjectInstanceRule = createRule("ObjectInstanceRule") "}" ]); -/** - * Link end rule. - * Matches an object reference with optional property specification. - */ export const LinkEndRule = createRule("LinkEndRule") .returns(LinkEnd) .as(({ set }) => [ @@ -118,35 +84,50 @@ export const LinkEndRule = createRule("LinkEndRule") optional(group(".", set("property", ref(AssociationEnd, ID)))) ]); -/** - * Link rule. - * Matches links between object instances: objectId1[.property] -- objectId2[.property] - */ export const LinkRule = createRule("LinkRule") .returns(Link) .as(({ set }) => [set("source", LinkEndRule), "--", set("target", LinkEndRule)]); -/** - * Metamodel file import rule. - * Matches "using" statements for importing metamodel files. - */ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") .returns(MetamodelFileImport) .as(({ set }) => ["using", set("file", STRING)]); /** - * Model root rule. - * Matches the complete model file structure. + * Default root rule for the Model language (without contributed imports). + * Use createModelRule() with resolved plugins for the full grammar. */ export const ModelRule = createRule("ModelRule") .returns(Model) .as(({ add, set }) => [ many(NEWLINE), set("import", MetamodelFileImportRule), + many(NEWLINE), many(or(add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) ]); /** - * Additional terminals for the Model language. + * Builds the root Model rule, alternating between hand-authored objects/links + * and any data imports contributed by plugins (e.g. CSV). + * + * @param contributedImportRules Wrapper rules for imports contributed by resolved plugins + * @returns The root parser rule for the Model language */ +export function createModelRule(contributedImportRules: ParserRule[] = []): ParserRule { + if (contributedImportRules.length === 0) { + return ModelRule; + } + + return createRule("ModelRule") + .returns(Model) + .as(({ add, set }) => { + const importAlternatives: RuleEntry[] = contributedImportRules.map((rule) => add("imports", rule)); + return [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or(...importAlternatives, add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) + ]; + }); +} + export const ModelTerminals = [WS, HIDDEN_NEWLINE, ML_COMMENT, SL_COMMENT]; diff --git a/app/packages/language-model/src/grammar/modelTypes.ts b/app/packages/language-model/src/grammar/modelTypes.ts index b0d7c00a..feecaec1 100644 --- a/app/packages/language-model/src/grammar/modelTypes.ts +++ b/app/packages/language-model/src/grammar/modelTypes.ts @@ -137,12 +137,26 @@ export const MetamodelFileImport = createInterface("MetamodelFileImport").attrs( */ export type MetamodelFileImportType = ASTType; +/** + * Base interface for data imports contributed by plugins (e.g. CSV, and future formats). + * A contribution plugin wraps its own import block in an interface extending this one, + * the same way config sections extend `BaseConfigSection`. + */ +export const BaseModelImport = createInterface("BaseModelImport").attrs({}); + +/** + * Type representing a BaseModelImport AST node. + */ +export type BaseModelImportType = ASTType; + /** * Model root interface. - * Contains metamodel imports, object instances, and links. + * Contains metamodel import, any number of plugin-contributed data imports (e.g. CSV), + * and hand-authored objects and links. */ export const Model = createInterface("Model").attrs({ import: MetamodelFileImport, + imports: [BaseModelImport], objects: [ObjectInstance], links: [Link] }); diff --git a/app/packages/language-model/src/index.ts b/app/packages/language-model/src/index.ts index 4b4ba414..d065aa7f 100644 --- a/app/packages/language-model/src/index.ts +++ b/app/packages/language-model/src/index.ts @@ -53,3 +53,5 @@ export * from "./validation/baseModelValidator.js"; export * from "./validation/modelValidator.js"; export * from "./generatedModelPlugin.js"; export * from "./modelPlugin.js"; +export * from "./plugin/modelContributionPlugin.js"; +export * from "./plugin/resolvePlugins.js"; diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index 67dec144..5d8f2529 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -1,9 +1,17 @@ import { type ExternalReferenceAdditionalServices, type LangiumLanguagePlugin, - type LangiumLanguagePluginProvider + type LangiumLanguagePluginProvider, + GrammarDeserializationContext, + ID, + NEWLINE, + HIDDEN_NEWLINE, + INT, + FLOAT, + STRING } from "@mdeo/language-common"; -import { ModelRule, ModelTerminals } from "./grammar/modelRules.js"; +import { Class } from "@mdeo/language-metamodel"; +import { createModelRule, ModelTerminals } from "./grammar/modelRules.js"; import { addExternalReferenceCollectionPhase, type ActionHandlerRegistryAdditionalServices, @@ -23,20 +31,38 @@ import { registerModelSerializers } from "./features/modelSerializers.js"; import { ModelDiagramModule } from "./features/diagram-server/modelDiagramModule.js"; import { registerModelValidationChecks } from "./validation/modelValidator.js"; import { ModelCompletionProvider } from "./features/modelCompletionProvider.js"; +import { ModelContributionPlugin } from "./plugin/modelContributionPlugin.js"; +import { resolveModelPlugins } from "./plugin/resolvePlugins.js"; +import type { ServerContributionPlugin } from "@mdeo/plugin"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; -/** - * The plugin for the Model language. - * Provides minimal language server functionality for .m files. - */ -function createModelPlugin(languageJsUrl?: string): LangiumLanguagePlugin { +function createModelPlugin( + contributionPlugins: ServerContributionPlugin[], + languageJsUrl?: string +): LangiumLanguagePlugin { + const modelPlugins = contributionPlugins.filter(ModelContributionPlugin.is); + + const deserializationContext = GrammarDeserializationContext.create( + [Class], + [], + [ID, NEWLINE, HIDDEN_NEWLINE, INT, FLOAT, STRING] + ); + + const resolvedPlugins = resolveModelPlugins(modelPlugins, deserializationContext); + + const ModelRule = createModelRule(resolvedPlugins.rules); + return { rootRule: ModelRule, additionalTerminals: ModelTerminals, module: { parser: { - TokenBuilder: () => new NewlineAwareTokenBuilder(new Set(["{"]), new Set(["("]), new Set(["}", ")"])), + TokenBuilder: () => new NewlineAwareTokenBuilder( + new Set(["{"]), + new Set(["("]), + new Set(["}", ")"]) + ), ValueConverter: () => new IdValueConverter(), ParserConfig: () => ({ maxLookahead: 4 @@ -73,11 +99,8 @@ function createModelPlugin(languageJsUrl?: string): LangiumLanguagePlugin = { - create(_contributionPlugins, languageJsUrl): LangiumLanguagePlugin { - return createModelPlugin(languageJsUrl); + create(contributionPlugins, languageJsUrl): LangiumLanguagePlugin { + return createModelPlugin(contributionPlugins, languageJsUrl); } }; diff --git a/app/packages/language-model/src/plugin/modelContributionPlugin.ts b/app/packages/language-model/src/plugin/modelContributionPlugin.ts new file mode 100644 index 00000000..d3bf2880 --- /dev/null +++ b/app/packages/language-model/src/plugin/modelContributionPlugin.ts @@ -0,0 +1,87 @@ +import type { ServerContributionPlugin } from "@mdeo/plugin"; +import type { SerializedGrammar } from "@mdeo/language-common"; + +/** + * A data import format that can be contributed by a plugin (e.g. CSV). + * Each plugin contributes exactly one import keyword and the rule/interface + * in its own serialized grammar that defines the content following that keyword. + */ +export interface ModelImportContribution { + /** + * The keyword introducing this import in the model file syntax (e.g. "CSV"), + * used as `import { ... }`. + */ + name: string; + + /** + * The name of the parser rule in the plugin's serialized grammar that defines + * the content of this import block (everything after the `{`). + */ + ruleName: string; + + /** + * The name of the interface in the plugin's serialized grammar that defines + * this import's AST type. + */ + interfaceName: string; +} + +/** + * Plugin for contributing a data import format to the Model language. + */ +export interface ModelContributionPlugin extends ServerContributionPlugin { + /** + * Identifies the plugin as a Model language contribution. + */ + type: typeof ModelContributionPlugin.TYPE; + + /** + * The short name of the plugin (e.g. "csv"). + */ + name: string; + + /** + * The language key used to get language services for this plugin. + * This is the language ID registered with Langium's ServiceRegistry. + */ + languageKey: string; + + /** + * The serialized grammar containing all rules for this plugin's import. + */ + grammar: SerializedGrammar; + + /** + * The import format contributed by this plugin. + */ + imports: ModelImportContribution[]; + + /** + * Plugin IDs that this plugin depends on. + * These plugins must be loaded before this one, and their exported types + * will be available in this plugin's deserialization context. + */ + dependencies: string[]; + + /** + * Names of types/interfaces exported by this plugin for use by other plugins. + */ + exportedTypes: string[]; +} + +export namespace ModelContributionPlugin { + /** + * The type identifier for ModelContributionPlugin. + */ + export const TYPE = "model-language-contribution"; + + /** + * Type guard for ModelContributionPlugin. + * + * @param value The value to check + * @returns True if the value is a ModelContributionPlugin, false otherwise + */ + export function is(value: ServerContributionPlugin): value is ModelContributionPlugin { + return "type" in value && value.type === TYPE; + } +} diff --git a/app/packages/language-model/src/plugin/resolvePlugins.ts b/app/packages/language-model/src/plugin/resolvePlugins.ts new file mode 100644 index 00000000..04315c8b --- /dev/null +++ b/app/packages/language-model/src/plugin/resolvePlugins.ts @@ -0,0 +1,369 @@ +import { + GrammarDeserializer, + isTerminalRule, + GrammarDeserializationContext, + type Interface, + type ParserRule, + type Type, + type TerminalRule, + createInterface, + createRule, + or, + many, + NEWLINE +} from "@mdeo/language-common"; +import type { ModelContributionPlugin, ModelImportContribution } from "./modelContributionPlugin.js"; +import { BaseModelImport, Model } from "../grammar/modelTypes.js"; +import { MetamodelFileImportRule } from "../grammar/modelRules.js"; +import type { AstNode } from "langium"; + +/** + * Naming and grammar information for a resolved import contribution. + */ +export interface ImportNamingInfo { + /** + * The keyword introducing this import (e.g. "CSV"). + */ + importName: string; + + /** + * The plugin that contributes this import. + */ + plugin: ModelContributionPlugin; + + /** + * The resolved wrapper interface for this import, extending BaseModelImport. + */ + interface: Interface; + + /** + * The resolved wrapper parser rule for this import. + */ + rule: ParserRule; +} + +/** + * Resolved plugins containing the contributions of all plugins. + */ +export interface ResolvedModelContributionPlugins { + /** + * All import naming info, indexed by the import keyword. + */ + imports: Map; + + /** + * Keywords introduced by resolved plugins. + */ + keywords: string[]; + + /** + * All wrapper parser rules from all plugins. + */ + rules: ParserRule[]; +} + +/** + * Sorts plugins by dependency order using topological sort. + * + * @param plugins The contribution plugins to sort + * @returns The sorted plugins + * @throws Error if circular dependencies are detected + */ +function sortPluginsByDependencies(plugins: ModelContributionPlugin[]): ModelContributionPlugin[] { + const sorted: ModelContributionPlugin[] = []; + const visited = new Set(); + const visiting = new Set(); + const pluginMap = new Map(); + + for (const plugin of plugins) { + pluginMap.set(plugin.id, plugin); + } + + function visit(pluginId: string): void { + if (visited.has(pluginId)) { + return; + } + if (visiting.has(pluginId)) { + throw new Error(`Circular dependency detected involving plugin: ${pluginId}`); + } + + visiting.add(pluginId); + + const plugin = pluginMap.get(pluginId); + if (plugin) { + for (const depId of plugin.dependencies) { + const depPlugin = pluginMap.get(depId); + if (!depPlugin) { + throw new Error(`Plugin '${pluginId}' depends on '${depId}' which is not available`); + } + visit(depId); + } + visited.add(pluginId); + visiting.delete(pluginId); + sorted.push(plugin); + } + } + + for (const plugin of plugins) { + visit(plugin.id); + } + + return sorted; +} + +/** + * Class responsible for resolving model contribution plugins. + * Encapsulates the state and logic for processing plugins and their imports. + */ +class ModelPluginResolver { + private readonly imports: Map; + private readonly keywords: string[]; + private readonly rules: ParserRule[]; + private readonly exportedTypes: (Interface | Type)[]; + private readonly baseTypes: (Interface | Type)[]; + private readonly baseRules: ParserRule[]; + private readonly baseTerminals: TerminalRule[]; + + constructor(deserializationContext: GrammarDeserializationContext) { + this.imports = new Map(); + this.keywords = []; + this.rules = []; + this.exportedTypes = []; + this.baseTypes = Array.from(deserializationContext.types?.values() ?? []); + this.baseRules = Array.from(deserializationContext.parserRules?.values() ?? []); + this.baseTerminals = Array.from(deserializationContext.terminalRules?.values() ?? []); + } + + /** + * Resolves all plugins and returns the combined result. + * + * @param sortedPlugins The plugins to process, sorted by dependency order + * @returns The resolved plugins with naming information + */ + resolve(sortedPlugins: ModelContributionPlugin[]): ResolvedModelContributionPlugins { + for (const plugin of sortedPlugins) { + this.processPlugin(plugin); + } + + return { + imports: this.imports, + keywords: this.keywords, + rules: this.rules + }; + } + + /** + * Processes a single plugin, deserializing its grammar and processing its import contribution. + * + * @param plugin The plugin to process + */ + private processPlugin(plugin: ModelContributionPlugin): void { + const pluginDeserializationContext = GrammarDeserializationContext.create( + [...this.baseTypes, ...this.exportedTypes], + this.baseRules, + this.baseTerminals + ); + + const deserializer = new GrammarDeserializer(plugin.grammar, pluginDeserializationContext); + const grammar = deserializer.deserializeGrammar(); + + const ruleMap = new Map>(); + for (const rule of grammar.rules) { + if (!isTerminalRule(rule)) { + ruleMap.set(rule.name, rule); + } + } + + const interfaceMap = new Map>(); + for (const iface of grammar.interfaces) { + interfaceMap.set(iface.name, iface); + } + + const typeMap = new Map>(); + for (const type of grammar.types) { + typeMap.set(type.name, type); + } + + for (const imp of plugin.imports) { + this.processImport(imp, plugin, ruleMap, interfaceMap); + } + + for (const exportedTypeName of plugin.exportedTypes) { + const exportedType = interfaceMap.get(exportedTypeName) ?? typeMap.get(exportedTypeName); + if (!exportedType) { + throw new Error( + `Plugin '${plugin.id}' exports type '${exportedTypeName}' which does not exist in its grammar.` + ); + } + this.exportedTypes.push(exportedType); + } + } + + /** + * Processes a single import contribution from a plugin. + * Creates the wrapper interface and rule, and registers the import's keyword. + * + * @param imp The import contribution to process + * @param plugin The plugin contributing the import + * @param ruleMap Map of rule names to rules in the plugin's grammar + * @param interfaceMap Map of interface names to interfaces in the plugin's grammar + */ + private processImport( + imp: ModelImportContribution, + plugin: ModelContributionPlugin, + ruleMap: Map>, + interfaceMap: Map> + ): void { + if (this.imports.has(imp.name)) { + const existing = this.imports.get(imp.name)!; + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' conflicts with the same import already contributed by plugin '${existing.plugin.id}'.` + ); + } + + const importRule = ruleMap.get(imp.ruleName); + if (importRule == undefined) { + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' references rule '${imp.ruleName}' which does not exist in the plugin's grammar.` + ); + } + + const importInterface = interfaceMap.get(imp.interfaceName); + if (importInterface == undefined) { + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' references interface '${imp.interfaceName}' which does not exist in the plugin's grammar.` + ); + } + + const { wrapperInterface, wrapperRule } = createImportWrapper(imp.name, importRule, importInterface); + + this.rules.push(wrapperRule); + this.imports.set(imp.name, { importName: imp.name, plugin, interface: wrapperInterface, rule: wrapperRule }); + this.keywords.push(imp.name); + } +} + +/** + * Creates the wrapper interface and rule for an import contribution. + * The wrapper wraps the import's content interface in a BaseModelImport, gated by its keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @param contentRule The parser rule for the import's content + * @param contentInterface The interface for the import's content + * @returns The wrapper interface and rule + */ +function createImportWrapper( + importName: string, + contentRule: ParserRule, + contentInterface: Interface +): { wrapperInterface: Interface; wrapperRule: ParserRule } { + const wrapperInterfaceName = getWrapperInterfaceName(importName); + const wrapperInterface = createInterface(wrapperInterfaceName).extends(BaseModelImport).attrs({ + content: contentInterface + }); + + const wrapperRuleName = getWrapperRuleName(importName); + const wrapperRule = createRule(wrapperRuleName) + .returns(wrapperInterface) + .as(({ set }) => ["import", importName, set("content", contentRule)]); + + return { wrapperInterface, wrapperRule }; +} + +/** + * Returns the wrapper interface name for a given import keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @returns The wrapper interface name (e.g. "ModelCsvImport") + */ +export function getWrapperInterfaceName(importName: string): string { + return `Model${capitalize(importName)}Import`; +} + +/** + * Returns the wrapper parser rule name for a given import keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @returns The wrapper rule name (e.g. "ModelCsvImportWrapper") + */ +export function getWrapperRuleName(importName: string): string { + return `Model${capitalize(importName)}ImportWrapper`; +} + +function capitalize(value: string): string { + return value.length === 0 ? value : value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); +} + +/** + * Generates the grammar for the standalone (artificial) language of an import contribution plugin. + * This is used by the plugin's own language service to parse partial model files that + * contain only this plugin's import block, alongside the base "using" metamodel import. + * + * @param plugin The contribution plugin whose standalone grammar is to be generated + * @param deserializationContext Context for resolving external type/rule references in the plugin's grammar + * @returns The root parser rule for the standalone grammar + */ +export function generateModelContributionGrammar( + plugin: ModelContributionPlugin, + deserializationContext: GrammarDeserializationContext +): ParserRule { + const deserializer = new GrammarDeserializer(plugin.grammar, deserializationContext); + const grammar = deserializer.deserializeGrammar(); + + const ruleMap = new Map>(); + for (const rule of grammar.rules) { + if (!isTerminalRule(rule)) { + ruleMap.set(rule.name, rule); + } + } + + const interfaceMap = new Map>(); + for (const iface of grammar.interfaces) { + interfaceMap.set(iface.name, iface); + } + + const wrapperRules: ParserRule[] = []; + + for (const imp of plugin.imports) { + const contentRule = ruleMap.get(imp.ruleName); + if (contentRule == undefined) { + throw new Error(`Import '${imp.name}' in plugin '${plugin.id}' references rule '${imp.ruleName}' which does not exist.`); + } + + const contentInterface = interfaceMap.get(imp.interfaceName); + if (contentInterface == undefined) { + throw new Error( + `Import '${imp.name}' in plugin '${plugin.id}' references interface '${imp.interfaceName}' which does not exist.` + ); + } + + const { wrapperRule } = createImportWrapper(imp.name, contentRule, contentInterface); + wrapperRules.push(wrapperRule); + } + + return createRule(`${capitalize(plugin.name)}PluginModelRule`) + .returns(Model) + .as(({ add, set }) => [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or(...wrapperRules.map((r) => add("imports", r)), NEWLINE)) + ]); +} + +/** + * Resolves model contribution plugins in dependency order, deserializing their + * grammars and wrapping each contributed import in a BaseModelImport-extending rule. + * + * @param plugins The contribution plugins + * @param deserializationContext The deserialization context for resolving external references + * @returns The resolved plugins with naming information + */ +export function resolveModelPlugins( + plugins: ModelContributionPlugin[], + deserializationContext: GrammarDeserializationContext +): ResolvedModelContributionPlugins { + const sortedPlugins = sortPluginsByDependencies(plugins); + const resolver = new ModelPluginResolver(deserializationContext); + return resolver.resolve(sortedPlugins); +} diff --git a/app/packages/language-shared/src/index.ts b/app/packages/language-shared/src/index.ts index 81dd82c0..39497703 100644 --- a/app/packages/language-shared/src/index.ts +++ b/app/packages/language-shared/src/index.ts @@ -79,6 +79,7 @@ export * from "./serialization/comments.js"; export * from "./serialization/defaultTokenSerializers.js"; export * from "./serialization/indentationGuesser.js"; export * from "./serialization/newlineSepSerializer.js"; +export * from "./util/csv.js"; export * from "./util/pathUtils.js"; export * from "./workspace/workspaceEditService.js"; export * from "./sharedImport.js"; diff --git a/app/packages/language-shared/src/util/csv.ts b/app/packages/language-shared/src/util/csv.ts new file mode 100644 index 00000000..cdf698f4 --- /dev/null +++ b/app/packages/language-shared/src/util/csv.ts @@ -0,0 +1,63 @@ +/** + * Parses CSV text into rows of fields, honouring RFC 4180 quoting: quoted fields + * may contain commas and newlines, and `""` inside a quoted field is an escaped + * literal quote. Blank lines are skipped. + * + * @param text The raw CSV file content + * @returns The parsed rows, each an array of field values + */ +export function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let currentRow: string[] = []; + let currentField = ""; + let inQuotes = false; + let i = 0; + + const endField = () => { + currentRow.push(currentField); + currentField = ""; + }; + + const endRow = () => { + endField(); + if (!(currentRow.length === 1 && currentRow[0].trim() === "")) { + rows.push(currentRow); + } + currentRow = []; + }; + + while (i < text.length) { + const c = text[i]; + if (inQuotes) { + if (c === '"') { + if (i + 1 < text.length && text[i + 1] === '"') { + currentField += '"'; + i++; + } else { + inQuotes = false; + } + } else { + currentField += c; + } + } else { + if (c === '"') { + inQuotes = true; + } else if (c === ",") { + endField(); + } else if (c === "\r") { + // skip + } else if (c === "\n") { + endRow(); + } else { + currentField += c; + } + } + i++; + } + + if (currentField !== "" || currentRow.length > 0) { + endRow(); + } + + return rows; +} diff --git a/app/packages/service-csv/package.json b/app/packages/service-csv/package.json new file mode 100644 index 00000000..6e2e9a87 --- /dev/null +++ b/app/packages/service-csv/package.json @@ -0,0 +1,26 @@ +{ + "name": "@mdeo/service-csv", + "version": "0.1.0", + "private": true, + "type": "module", + "files": ["dist", "src"], + "scripts": { + "build:static": "vite build", + "watch:static": "vite build --watch", + "start": "node dist/index.js", + "watch": "tsx watch src/index.ts" + }, + "dependencies": { + "@mdeo/language-csv": "^0.1.0", + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "tsx": "^4.21.0", + "vite": "^7.3.1" + } +} diff --git a/app/packages/service-csv/src/index.ts b/app/packages/service-csv/src/index.ts new file mode 100644 index 00000000..8d97c5f3 --- /dev/null +++ b/app/packages/service-csv/src/index.ts @@ -0,0 +1,67 @@ +import { Table } from "lucide"; +import { + startLanguageService, + parseServiceConfigFromEnv, + initializePluginContext, + AST_HANDLER_KEY, + astHandler, + type ServicePluginDefinition, + type LanguageServiceConfig +} from "@mdeo/service-common"; +import { + convertIcon, + defaultLanguageConfiguration, + defaultMonarchTokenProvider, + serializeMonarchTokensProvider, + type ExternalReferenceAdditionalServices +} from "@mdeo/language-common"; +import type { LanguagePlugin } from "@mdeo/plugin"; + +const csvLanguagePlugin: LanguagePlugin = { + id: "csv", + name: "CSV", + extension: ".csv", + newFileAction: true, + icon: convertIcon(Table), + serverPlugin: { + import: "language.js" + }, + graphicalEditorPlugin: undefined, + textualEditorPlugin: { + languageConfiguration: defaultLanguageConfiguration, + monarchTokensProvider: serializeMonarchTokensProvider({ + ...defaultMonarchTokenProvider, + keywords: [] + }) + }, + isGenerated: false +}; + +const csvServicePlugin: ServicePluginDefinition = { + id: "csv-service", + name: "CSV", + description: "Language support for CSV data files (.csv files)", + icon: convertIcon(Table), + languagePlugins: [csvLanguagePlugin], + contributionPlugins: [] +}; + +initializePluginContext(); + +const { csvPluginProvider } = await import("@mdeo/language-csv"); + +const envConfig = parseServiceConfigFromEnv(); + +const csvLanguageConfig: LanguageServiceConfig = { + languagePlugin: csvLanguagePlugin, + languagePluginProvider: csvPluginProvider, + fileDataHandlers: { + [AST_HANDLER_KEY]: astHandler + } +}; + +await startLanguageService({ + ...envConfig, + plugin: csvServicePlugin, + languages: [csvLanguageConfig] +}); diff --git a/app/packages/service-csv/src/served/language.ts b/app/packages/service-csv/src/served/language.ts new file mode 100644 index 00000000..89853725 --- /dev/null +++ b/app/packages/service-csv/src/served/language.ts @@ -0,0 +1,3 @@ +import { csvPluginProvider } from "@mdeo/language-csv"; + +export default csvPluginProvider; diff --git a/app/packages/service-csv/tsconfig.json b/app/packages/service-csv/tsconfig.json new file mode 100644 index 00000000..b3bc6233 --- /dev/null +++ b/app/packages/service-csv/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-csv" }, + { "path": "../language-common" }, + { "path": "../language-shared" }, + { "path": "../plugin" }, + { "path": "../service-common" } + ] +} diff --git a/app/packages/service-csv/vite.config.ts b/app/packages/service-csv/vite.config.ts new file mode 100644 index 00000000..1df064ba --- /dev/null +++ b/app/packages/service-csv/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vite"; +import { resolve } from "path"; + +export default defineConfig({ + build: { + lib: { + entry: { + language: resolve(__dirname, "src/served/language.ts") + }, + formats: ["es"] + }, + outDir: "static", + emptyOutDir: true, + sourcemap: true + } +}); diff --git a/app/packages/service-model-common/package.json b/app/packages/service-model-common/package.json new file mode 100644 index 00000000..4aa92fa2 --- /dev/null +++ b/app/packages/service-model-common/package.json @@ -0,0 +1,19 @@ +{ + "name": "@mdeo/service-model-common", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist", + "src" + ], + "dependencies": { + "@mdeo/language-model": "^0.1.0", + "@mdeo/service-common": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/service-model-common/src/index.ts b/app/packages/service-model-common/src/index.ts new file mode 100644 index 00000000..db0f69d5 --- /dev/null +++ b/app/packages/service-model-common/src/index.ts @@ -0,0 +1,2 @@ +export * from "./modelPluginHandler.js"; +export * from "./modelPluginTypes.js"; diff --git a/app/packages/service-model-common/src/modelPluginHandler.ts b/app/packages/service-model-common/src/modelPluginHandler.ts new file mode 100644 index 00000000..cbe6c655 --- /dev/null +++ b/app/packages/service-model-common/src/modelPluginHandler.ts @@ -0,0 +1,104 @@ +import { TextDocument } from "langium"; +import type { AstNode, LangiumDocument } from "langium"; +import { hasParserErrors } from "@mdeo/service-common"; +import type { RequestContext } from "@mdeo/service-common"; +import type { ModelPluginRequestBody } from "./modelPluginTypes.js"; + +/** + * A parsed import block belonging to the requesting plugin. + */ +export interface ModelPluginImport { + /** + * The wrapper node's `$type`, i.e. the wrapper interface name for the import keyword. + */ + type: string; + + /** + * The import block's content node, as defined by the plugin's own grammar. + */ + content: AstNode; +} + +/** + * Discriminated union returned by {@link buildModelPluginDocument}. + * + * - `"empty"` - the request carried no text, or the parsed document contained no + * imports; callers should return an empty but valid response. + * - `"error"` - the text could not be parsed; callers should return `data: null` + * while still reporting tracked dependencies. + * - `"success"` - the document parsed; callers can walk {@link imports}. + */ +export type ModelPluginDocumentResult = + | { type: "empty" } + | { type: "error" } + | { + type: "success"; + /** + * The built Langium document. + */ + document: LangiumDocument; + /** + * The import blocks contributed by this plugin. + */ + imports: ModelPluginImport[]; + /** + * The decoded request body supplied by the model service. + */ + requestBody: ModelPluginRequestBody; + }; + +/** + * Builds a partial model document for an import contribution plugin. + * + * Mirrors `buildConfigPluginDocument`, with one deliberate difference: only + * parser errors are treated as failures, not validation errors. A plugin's + * service has the plugin's own grammar but not the metamodel document, so + * cross-references into the metamodel cannot be linked here and would otherwise + * always report as errors. That costs nothing, because the model service has + * already validated the real `.m` document (including those references) before + * forwarding anything; plugins read referenced names from `$refText`. + * + * The synthetic document is given the originating `.m` file's URI so that paths + * written inside the import block resolve relative to the model file, exactly as + * they do in the model service. + * + * @param context The request context injected by the service framework + * @param languageKey The language ID of this contribution plugin's language + * @returns A discriminated {@link ModelPluginDocumentResult} + */ +export async function buildModelPluginDocument( + context: RequestContext, + languageKey: string +): Promise { + const requestBody = context.body as ModelPluginRequestBody; + const text = requestBody?.text ?? ""; + + if (!text.trim()) { + return { type: "empty" }; + } + + const textDocument = TextDocument.create(requestBody.modelFileUri, languageKey, 0, text); + context.services.shared.workspace.TextDocuments.set(textDocument); + const document = context.services.shared.workspace.LangiumDocumentFactory.fromTextDocument(textDocument); + + await context.services.shared.workspace.DocumentBuilder.build([document], { validation: false }); + if (hasParserErrors(document)) { + return { type: "error" }; + } + + const root = document.parseResult?.value as { imports?: AstNode[] } | undefined; + const imports = root?.imports; + if (root == undefined || !Array.isArray(imports) || imports.length === 0) { + return { type: "empty" }; + } + + return { + type: "success", + document, + imports: imports.map((wrapper) => ({ + type: wrapper.$type, + content: (wrapper as AstNode & { content: AstNode }).content + })), + requestBody + }; +} diff --git a/app/packages/service-model-common/src/modelPluginTypes.ts b/app/packages/service-model-common/src/modelPluginTypes.ts new file mode 100644 index 00000000..df87450d --- /dev/null +++ b/app/packages/service-model-common/src/modelPluginTypes.ts @@ -0,0 +1,139 @@ +import type { ModelDataInstance, ModelDataLink } from "@mdeo/language-model"; +import type { FileDependency, DataDependency } from "@mdeo/service-common"; + +/** + * Description of a single metamodel property, as sent to import plugins. + * + * Import plugins map their own source data (CSV columns, spreadsheet fields, and + * so on) onto these, so they need to know each property's name and type without + * having to resolve the metamodel document themselves. + */ +export interface MetamodelPropertyInfo { + /** + * Name of the property as declared in the metamodel. + */ + name: string; + + /** + * The property's declared type. `reference` means the property is an + * association end rather than an attribute. + */ + type: "string" | "int" | "long" | "double" | "float" | "boolean" | "enum" | "reference"; + + /** + * For enum-typed properties, the names of the enum's entries. + */ + enumEntries?: string[]; + + /** + * True when the property is an association end. + */ + isReference?: boolean; + + /** + * For reference properties, the name of the class being referenced. + */ + referencedClass?: string; +} + +/** + * Description of a single metamodel class, as sent to import plugins. + */ +export interface MetamodelClassInfo { + /** + * Name of the class as declared in the metamodel. + */ + name: string; + + /** + * The class' properties. + */ + properties: MetamodelPropertyInfo[]; +} + +/** + * Request body sent to a model import contribution plugin's request handler. + * + * Mirrors `ConfigPluginRequestBody`: the model service extracts the text of the + * plugin's own import block and hands it back to the plugin's service, so the + * plugin parses and interprets its own syntax rather than the model service + * knowing anything about it. + */ +export interface ModelPluginRequestBody { + /** + * The source text of this plugin's import block content, without the + * surrounding `import ` keywords. + */ + text: string; + + /** + * URI string of the originating `.m` file. Used to build a stable synthetic + * document URI and to resolve paths referenced by the import relative to it. + */ + modelFileUri: string; + + /** + * Path of the metamodel the model imports, as written in the `.m` file. + */ + metamodelPath: string; + + /** + * The metamodel's classes, so the plugin can map its source data onto them + * without resolving the metamodel document itself. + */ + metamodelClasses: MetamodelClassInfo[]; +} + +/** + * The instances and links produced by one import contribution plugin. + */ +export interface ModelPluginData { + /** + * Object instances derived from the plugin's imported data. + */ + instances: ModelDataInstance[]; + + /** + * Links derived from the plugin's imported data. + */ + links: ModelDataLink[]; + + /** + * Non-fatal problems encountered while importing (unknown columns, unresolved + * references, and similar). Present so the host service can surface them. + */ + warnings?: string[]; +} + +/** + * Response returned by a model import contribution plugin's request handler. + * + * As with the config equivalent, dependencies MUST be reported even when + * {@link data} is `null`, so the host service can propagate them to the cache + * and avoid incorrect cache invalidation. + * + * @template T The type of the payload. + */ +export interface ModelPluginRequestResponse { + /** + * The computed import data, or `null` when the plugin could not produce a + * valid result (for example a parse or validation error). + */ + data: T | null; + + /** + * Files (and their versions) read while processing the import. + */ + fileDependencies: FileDependency[]; + + /** + * Other file-data computations consulted while processing the import. + */ + dataDependencies: DataDependency[]; +} + +/** + * Request-handler key used by model import contribution plugin services to + * register and handle import-data requests from the model service. + */ +export const MODEL_PLUGIN_REQUEST_KEY = "model-import"; diff --git a/app/packages/service-model-common/tsconfig.json b/app/packages/service-model-common/tsconfig.json new file mode 100644 index 00000000..c127f254 --- /dev/null +++ b/app/packages/service-model-common/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + { + "path": "../plugin" + }, + { + "path": "../language-common" + }, + { + "path": "../language-model" + }, + { + "path": "../service-common" + } + ], + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/app/packages/service-model-csv/package.json b/app/packages/service-model-csv/package.json new file mode 100644 index 00000000..9b273838 --- /dev/null +++ b/app/packages/service-model-csv/package.json @@ -0,0 +1,34 @@ +{ + "name": "@mdeo/service-model-csv", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist", + "src" + ], + "scripts": { + "build:static": "vite build", + "watch:static": "vite build --watch", + "start": "node dist/index.js", + "watch": "tsx watch src/index.ts" + }, + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-model-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1", + "tsx": "~4.21.0", + "vite": "~7.3.1" + } +} diff --git a/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts b/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts new file mode 100644 index 00000000..4430da9e --- /dev/null +++ b/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts @@ -0,0 +1,63 @@ +import type { RequestHandler } from "@mdeo/service-common"; +import { resolveRelativePath } from "@mdeo/language-shared"; +import { + buildModelPluginDocument, + type ModelPluginData, + type ModelPluginRequestResponse +} from "@mdeo/service-model-common"; +import { importCsvEntries, MODEL_CSV_LANGUAGE_KEY } from "@mdeo/language-model-csv"; +import type { CsvImportBlockType, ModelCsvServices } from "@mdeo/language-model-csv"; + +/** + * Handles import-data requests forwarded by the model service. + * + * The model service does not know anything about CSV: it hands over the text of + * the `import CSV { ... }` block together with a description of the metamodel's + * classes, and this handler parses that text with the CSV grammar, reads the + * referenced CSV files, and returns the resulting instances and links. + * + * Class names are read from `$refText` rather than a resolved reference, because + * the metamodel document is not loaded in this service. The model service has + * already validated those references against the real metamodel before + * forwarding, so the names are known to be valid here. + */ +export const csvImportRequestHandler: RequestHandler = async ( + context +): Promise => { + const result = await buildModelPluginDocument(context, MODEL_CSV_LANGUAGE_KEY); + + if (result.type === "empty") { + return { data: { instances: [], links: [] }, ...context.serverApi.getTrackedRequests() }; + } + if (result.type === "error") { + return { data: null, ...context.serverApi.getTrackedRequests() }; + } + + const { document, imports, requestBody } = result; + + const entries = []; + for (const importBlock of imports) { + const content = importBlock.content as CsvImportBlockType; + for (const entry of content.imports ?? []) { + const file = entry.file ?? ""; + if (file === "") { + continue; + } + const csvPath = resolveRelativePath(document, file).path; + const { content: csvText } = await context.serverApi.readFile(csvPath); + entries.push({ + className: entry.class?.$refText ?? "", + csvText, + mappings: (entry.mappings ?? []).map((mapping) => ({ + csvColumn: mapping.csvColumn, + property: mapping.property + })) + }); + } + } + + const { instances, links, warnings } = importCsvEntries(entries, requestBody.metamodelClasses); + const data: ModelPluginData = { instances, links, warnings }; + + return { data, ...context.serverApi.getTrackedRequests() }; +}; diff --git a/app/packages/service-model-csv/src/index.ts b/app/packages/service-model-csv/src/index.ts new file mode 100644 index 00000000..be4532e3 --- /dev/null +++ b/app/packages/service-model-csv/src/index.ts @@ -0,0 +1,83 @@ +import { Table } from "lucide"; +import { convertIcon } from "@mdeo/language-common"; +import type { ModelCsvServices } from "@mdeo/language-model-csv"; +import { + parseServiceConfigFromEnv, + type ServiceConfig, + type ServicePluginDefinition, + type LanguageServiceConfig, + initializePluginContext, + astHandler, + AST_HANDLER_KEY, + startLanguageService +} from "@mdeo/service-common"; +import { MODEL_PLUGIN_REQUEST_KEY } from "@mdeo/service-model-common"; +import type { LanguagePlugin } from "@mdeo/plugin"; + +const icon = convertIcon(Table); + +/** + * Language plugin definition for the model-csv language. + * This is a generated language that provides services for the CSV import contribution. + */ +const modelCsvLanguagePlugin: LanguagePlugin = { + id: "model-csv", + name: "Model CSV", + extension: undefined, + newFileAction: false, + icon, + serverPlugin: { + import: "language.js" + }, + graphicalEditorPlugin: undefined, + textualEditorPlugin: undefined, + isGenerated: true +}; + +initializePluginContext(); + +const { modelCsvPluginProvider, createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); +const { csvImportRequestHandler } = await import("./handler/csvImportRequestHandler.js"); + +/** + * Plugin definition for the model-csv service. + */ +const modelCsvServicePlugin: ServicePluginDefinition = { + id: "model-csv-service", + name: "Model CSV", + description: "Language support for the CSV import contribution to the model language", + icon, + languagePlugins: [modelCsvLanguagePlugin], + contributionPlugins: [ + { + languageId: "model", + description: "Provides import CSV { ClassName from \"file.csv\" { \"column\" = property } } syntax for the model language", + additionalKeywords: ["import", "CSV", "from"], + serverContributionPlugins: [createModelCsvContributionPlugin()] + } + ] +}; + +const envConfig = parseServiceConfigFromEnv(); + +/** + * Language configuration for the model-csv language. + */ +const modelCsvLanguageConfig: LanguageServiceConfig = { + languagePlugin: modelCsvLanguagePlugin, + languagePluginProvider: modelCsvPluginProvider, + fileDataHandlers: { + [AST_HANDLER_KEY]: astHandler + }, + requestHandlers: { + [MODEL_PLUGIN_REQUEST_KEY]: csvImportRequestHandler + } +}; + +const config: ServiceConfig = { + ...envConfig, + plugin: modelCsvServicePlugin, + languages: [modelCsvLanguageConfig] +}; + +await startLanguageService(config); diff --git a/app/packages/service-model-csv/src/served/language.ts b/app/packages/service-model-csv/src/served/language.ts new file mode 100644 index 00000000..f0bfae63 --- /dev/null +++ b/app/packages/service-model-csv/src/served/language.ts @@ -0,0 +1,7 @@ +/** + * Re-exports the model-csv language plugin provider for the language server. + * This file is served as an ESM module for the workbench language server to import. + */ +import { modelCsvPluginProvider } from "@mdeo/language-model-csv"; + +export default modelCsvPluginProvider; diff --git a/app/packages/service-model-csv/tsconfig.json b/app/packages/service-model-csv/tsconfig.json new file mode 100644 index 00000000..d169b31b --- /dev/null +++ b/app/packages/service-model-csv/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + { "path": "../plugin" }, + { "path": "../language-common" }, + { "path": "../language-model" }, + { "path": "../language-model-csv" }, + { "path": "../language-shared" }, + { "path": "../service-common" }, + { "path": "../service-model-common" } + ], + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/app/packages/service-model-csv/vite.config.ts b/app/packages/service-model-csv/vite.config.ts new file mode 100644 index 00000000..1df064ba --- /dev/null +++ b/app/packages/service-model-csv/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vite"; +import { resolve } from "path"; + +export default defineConfig({ + build: { + lib: { + entry: { + language: resolve(__dirname, "src/served/language.ts") + }, + formats: ["es"] + }, + outDir: "static", + emptyOutDir: true, + sourcemap: true + } +}); diff --git a/app/packages/service-model/package.json b/app/packages/service-model/package.json index a1e15b88..95210e02 100644 --- a/app/packages/service-model/package.json +++ b/app/packages/service-model/package.json @@ -17,9 +17,11 @@ "@mdeo/editor-model": "^0.1.0", "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", "lucide": "^0.575.0" }, "devDependencies": { diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index 3bbc6ff8..b131bdea 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -1,22 +1,76 @@ -import { type ModelData, type ModelServices, ModelDataConverter, Model } from "@mdeo/language-model"; -import { hasErrors, type FileDataHandler } from "@mdeo/service-common"; +import { + type ModelData, + type ModelServices, + ModelDataConverter, + Model, + ModelContributionPlugin, + getWrapperInterfaceName +} from "@mdeo/language-model"; +import type { + MetamodelClassInfo, + MetamodelPropertyInfo, + ModelPluginData, + ModelPluginRequestBody, + ModelPluginRequestResponse +} from "@mdeo/service-model-common"; +import { MODEL_PLUGIN_REQUEST_KEY } from "@mdeo/service-model-common"; +import { hasErrors, type FileDataHandler, type FileDependency, type DataDependency } from "@mdeo/service-common"; +import { resolveRelativePath } from "@mdeo/language-shared"; +import type { AstNode } from "langium"; + +export const MODEL_DATA_HANDLER_KEY = "model-data"; /** - * Key for the model data handler. + * Extracts a flat description of a metamodel's classes for import plugins. + * + * Import plugins map their own source data onto the metamodel but have no access + * to the metamodel document, so this is sent to them as plain data. + * + * @param metamodelAst The parsed metamodel document root + * @returns One entry per class declared in the metamodel */ -export const MODEL_DATA_HANDLER_KEY = "model-data"; +function extractMetamodelClasses(metamodelAst: { elements?: unknown[] } | undefined): MetamodelClassInfo[] { + return ((metamodelAst?.elements ?? []) as any[]) + .filter((element) => element.$type === "Class") + .map( + (cls): MetamodelClassInfo => ({ + name: cls.name, + properties: (cls.properties ?? []).map((prop: any): MetamodelPropertyInfo => { + const primitiveType = prop.type?.name; + const enumRef = prop.type?.enum?.ref; + const isReference = prop.$type === "AssociationEnd"; + return { + name: prop.name, + type: isReference + ? "reference" + : enumRef + ? "enum" + : ((primitiveType ?? "string") as MetamodelPropertyInfo["type"]), + enumEntries: enumRef ? enumRef.entries?.map((entry: any) => entry.name) : undefined, + isReference, + referencedClass: isReference ? prop.class?.ref?.name : undefined + }; + }) + }) + ); +} /** - * Handler for computing the ModelData representation of a model file. - * Converts the language AST into a lightweight format suitable for serialization. + * File-data handler that computes the structured data for a model file. * - * Returns null if the document has errors to ensure only valid models are returned. + * Hand-authored objects and links are converted directly. Imported data is not: + * the model service has no knowledge of any particular import format, so for + * each contribution plugin whose import appears in the file it forwards that + * import block's own source text to the plugin's language service and merges + * back the instances and links the plugin returns. This mirrors how the config + * service delegates each section to the plugin that contributes it, and keeps + * the model service independent of CSV or any other future import format. * - * @param context The file data context with path, content, and services - * @returns Promise resolving to the file data result with ModelData or null + * Returns `null` when the document has errors, or when any plugin reports that + * it could not produce a valid result. */ export const modelDataHandler: FileDataHandler = async (context) => { - const { instance, fileInfo, serverApi } = context; + const { instance, fileInfo, serverApi, contributionPlugins } = context; if (fileInfo == undefined) { return { @@ -42,10 +96,95 @@ export const modelDataHandler: FileDataHandler } const converter = new ModelDataConverter(reflection); - const modelData = converter.convertModel(model); + const handAuthored = converter.convertModel(model); + + const modelImports = (model.imports ?? []) as (AstNode & { content?: AstNode })[]; + const activeModelPlugins = contributionPlugins.filter(ModelContributionPlugin.is); + + if (modelImports.length === 0 || activeModelPlugins.length === 0) { + return { + data: handAuthored, + ...serverApi.getTrackedRequests() + }; + } + + const metamodelPath = model.import?.file; + if (!metamodelPath) { + return { data: null, ...serverApi.getTrackedRequests() }; + } + + const metamodelUri = resolveRelativePath(document, metamodelPath); + const metamodelDoc = instance.services.shared.workspace.LangiumDocuments.getDocument(metamodelUri); + if (!metamodelDoc) { + return { data: null, ...serverApi.getTrackedRequests() }; + } + + const metamodelClasses = extractMetamodelClasses(metamodelDoc.parseResult?.value as { elements?: unknown[] }); + + const importedInstances: ModelData["instances"] = []; + const importedLinks: ModelData["links"] = []; + const accumulatedFileDeps: FileDependency[] = []; + const accumulatedDataDeps: DataDependency[] = []; + + for (const plugin of activeModelPlugins) { + const partialBlocks: string[] = []; + for (const contributedImport of plugin.imports) { + const wrapperType = getWrapperInterfaceName(contributedImport.name); + for (const modelImport of modelImports) { + if (modelImport.$type !== wrapperType) { + continue; + } + const contentText = modelImport.content?.$cstNode?.text ?? ""; + if (contentText !== "") { + partialBlocks.push(`import ${contributedImport.name} ${contentText}`); + } + } + } + + if (partialBlocks.length === 0) { + continue; + } + + const requestBody: ModelPluginRequestBody = { + text: `using "${metamodelPath}"\n\n${partialBlocks.join("\n")}`, + modelFileUri: fileInfo.uri.toString(), + metamodelPath, + metamodelClasses + }; + + const rawPluginResult = await serverApi.sendPluginRequest( + plugin.languageKey, + MODEL_PLUGIN_REQUEST_KEY, + requestBody + ); + const pluginResponse = rawPluginResult as ModelPluginRequestResponse; + + accumulatedFileDeps.push(...(pluginResponse?.fileDependencies ?? [])); + accumulatedDataDeps.push(...(pluginResponse?.dataDependencies ?? [])); + + if (pluginResponse?.data == null) { + const trackedRequests = serverApi.getTrackedRequests(); + return { + data: null, + fileDependencies: [...trackedRequests.fileDependencies, ...accumulatedFileDeps], + dataDependencies: [...trackedRequests.dataDependencies, ...accumulatedDataDeps] + }; + } + + importedInstances.push(...(pluginResponse.data.instances ?? [])); + importedLinks.push(...(pluginResponse.data.links ?? [])); + } + + const modelData: ModelData = { + metamodelPath, + instances: [...handAuthored.instances, ...importedInstances], + links: [...handAuthored.links, ...importedLinks] + }; + const trackedRequests = serverApi.getTrackedRequests(); return { data: modelData, - ...serverApi.getTrackedRequests() + fileDependencies: [...trackedRequests.fileDependencies, ...accumulatedFileDeps], + dataDependencies: [...trackedRequests.dataDependencies, ...accumulatedDataDeps] }; }; diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index aee5db1d..6c4df970 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -1,47 +1,28 @@ -import { Network } from "lucide"; +import { BookOpen } from "lucide"; import { defaultLanguageConfiguration, defaultMonarchTokenProvider, serializeMonarchTokensProvider, - type ActionIconNode + convertIcon } from "@mdeo/language-common"; import { + startLanguageService, parseServiceConfigFromEnv, type ServiceConfig, type ServicePluginDefinition, type LanguageServiceConfig, initializePluginContext, astHandler, - AST_HANDLER_KEY, - startLanguageService + AST_HANDLER_KEY } from "@mdeo/service-common"; -import { convertIcon } from "@mdeo/language-common"; import type { ModelServices, GeneratedModelServices } from "@mdeo/language-model"; import type { LanguagePlugin } from "@mdeo/plugin"; -const icon: ActionIconNode = convertIcon(Network).map((entry) => { - if (entry[0] === "rect") { - return [ - "rect", - { - ...entry[1], - fill: "currentColor" - } - ]; - } else { - return entry; - } -}); - -/** - * Language plugin definition for the model language. - */ const modelLanguagePlugin: LanguagePlugin = { id: "model", name: "Model", extension: ".m", - newFileAction: true, - icon, + icon: convertIcon(BookOpen), serverPlugin: { import: "language.js" }, @@ -52,6 +33,9 @@ const modelLanguagePlugin: LanguagePlugin = { }, textualEditorPlugin: { languageConfiguration: defaultLanguageConfiguration, + // Only the base language's own keyword. Keywords for contributed imports + // (e.g. `import CSV from`) come from each contribution plugin's + // additionalKeywords and are merged in by the workbench. monarchTokensProvider: serializeMonarchTokensProvider({ ...defaultMonarchTokenProvider, keywords: ["using"] @@ -61,14 +45,14 @@ const modelLanguagePlugin: LanguagePlugin = { }; /** - * Language plugin definition for the generated model language. + * Language plugin definition for the generated model language (.m_gen), e.g. optimizer solutions. */ const generatedModelLanguagePlugin: LanguagePlugin = { id: "model_gen", name: "Generated Model", extension: ".m_gen", newFileAction: false, - icon, + icon: convertIcon(BookOpen), serverPlugin: { import: "generatedLanguage.js" }, @@ -81,18 +65,6 @@ const generatedModelLanguagePlugin: LanguagePlugin = { isGenerated: true }; -/** - * Plugin definition for the model service. - */ -const modelServicePlugin: ServicePluginDefinition = { - id: "model-service", - name: "Model", - description: "Language support for model definitions (.m and .m_gen files)", - icon, - languagePlugins: [modelLanguagePlugin, generatedModelLanguagePlugin], - contributionPlugins: [] -}; - initializePluginContext(); const { modelPluginProvider } = await import("@mdeo/language-model"); @@ -101,9 +73,15 @@ const { modelDataHandler, MODEL_DATA_HANDLER_KEY } = await import("./handler/mod const envConfig = parseServiceConfigFromEnv(); -/** - * Language configuration for the model language. - */ +const modelServicePlugin: ServicePluginDefinition = { + id: "model-service", + name: "Model", + description: "Language support for model definitions (.m and .m_gen files)", + icon: convertIcon(BookOpen), + languagePlugins: [modelLanguagePlugin, generatedModelLanguagePlugin], + contributionPlugins: [] +}; + const modelLanguageConfig: LanguageServiceConfig = { languagePlugin: modelLanguagePlugin, languagePluginProvider: modelPluginProvider, @@ -113,9 +91,6 @@ const modelLanguageConfig: LanguageServiceConfig = { } }; -/** - * Language configuration for the generated model language. - */ const generatedModelLanguageConfig: LanguageServiceConfig = { languagePlugin: generatedModelLanguagePlugin, languagePluginProvider: generatedModelPluginProvider, diff --git a/app/packages/service-model/tsconfig.json b/app/packages/service-model/tsconfig.json index 94acbe66..aad7cf23 100644 --- a/app/packages/service-model/tsconfig.json +++ b/app/packages/service-model/tsconfig.json @@ -19,8 +19,16 @@ }, { "path": "../service-common" + }, + { + "path": "../service-model-common" } ], - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "dist", + "node_modules" + ] +} \ No newline at end of file diff --git a/app/packages/workbench/vite.config.ts b/app/packages/workbench/vite.config.ts index 1de70a51..5d4690cf 100644 --- a/app/packages/workbench/vite.config.ts +++ b/app/packages/workbench/vite.config.ts @@ -27,6 +27,10 @@ export default defineConfig({ "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp" }, + // Matched by prefix in declaration order, so a longer path has to come + // before any shorter path it starts with: model-transformation and + // model-csv before model, config-optimization and config-mdeo before + // config, and plain csv last. proxy: { "/plugin/model-transformation": { target: "http://localhost:3003", @@ -36,6 +40,14 @@ export default defineConfig({ rewrite: (path) => path.replace(/^\/plugin\/model-transformation/, ""), configure: addCoopCoepHeaders }, + "/plugin/model-csv": { + target: "http://localhost:3007", + changeOrigin: true, + secure: false, + ws: true, + rewrite: (path) => path.replace(/^\/plugin\/model-csv/, ""), + configure: addCoopCoepHeaders + }, "/plugin/metamodel": { target: "http://localhost:3000", changeOrigin: true, @@ -84,6 +96,14 @@ export default defineConfig({ rewrite: (path) => path.replace(/^\/plugin\/config/, ""), configure: addCoopCoepHeaders }, + "/plugin/csv": { + target: "http://localhost:3008", + changeOrigin: true, + secure: false, + ws: true, + rewrite: (path) => path.replace(/^\/plugin\/csv/, ""), + configure: addCoopCoepHeaders + }, "/api": { target: "http://localhost:8080", changeOrigin: true, diff --git a/app/tsconfig.build.json b/app/tsconfig.build.json index 42fc189f..c0bef162 100644 --- a/app/tsconfig.build.json +++ b/app/tsconfig.build.json @@ -5,6 +5,12 @@ { "path": "packages/plugin" }, + { + "path": "packages/language-csv" + }, + { + "path": "packages/service-csv" + }, { "path": "packages/language-common" }, @@ -32,6 +38,9 @@ { "path": "packages/language-model" }, + { + "path": "packages/language-model-csv" + }, { "path": "packages/language-model-transformation" }, @@ -83,6 +92,12 @@ { "path": "packages/service-model" }, + { + "path": "packages/service-model-common" + }, + { + "path": "packages/service-model-csv" + }, { "path": "packages/service-script" }, diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index 84c897b3..8681056c 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -87,7 +87,7 @@ services: ADMIN_PASSWORD: admin PLUGIN_BASE_URL: "" INTERNAL_PLUGIN_BASE_URL: "http://workbench:80" - DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo" + DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo,/plugin/csv,/plugin/model-csv" depends_on: postgres-backend: condition: service_healthy @@ -108,6 +108,8 @@ services: PLUGIN_CONFIG_OPTIMIZATION_SERVICE: http://service-config-optimization:3000 PLUGIN_CONFIG_MDEO_SERVICE: http://service-config-mdeo:3000 BACKEND_SERVICE: http://backend:8080 + PLUGIN_CSV_SERVICE: http://service-csv:3000 + PLUGIN_MODEL_CSV_SERVICE: http://service-model-csv:3000 ports: - "4242:80" depends_on: @@ -125,6 +127,10 @@ services: condition: service_healthy service-config-mdeo: condition: service_healthy + service-model-csv: + condition: service_healthy + service-csv: + condition: service_healthy restart: unless-stopped service-metamodel: @@ -148,6 +154,23 @@ services: start_period: 30s restart: unless-stopped + service-csv: + build: + context: .. + dockerfile: infra/docker/service-csv/Dockerfile + environment: + PORT: "3000" + HOST: "0.0.0.0" + BACKEND_API_URL: "http://backend:8080/api" + MAX_LANGIUM_INSTANCES: "5" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + service-model: build: context: .. @@ -169,6 +192,27 @@ services: start_period: 30s restart: unless-stopped + service-model-csv: + build: + context: .. + dockerfile: infra/docker/service-model-csv/Dockerfile + args: + SERVICE_VERSION: ${SERVICE_VERSION:-} + ports: + - "3007:3000" + environment: + PORT: "3000" + HOST: "0.0.0.0" + BACKEND_API_URL: "http://backend:8080/api" + MAX_LANGIUM_INSTANCES: "5" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + service-script: build: context: .. @@ -310,7 +354,7 @@ services: args: SERVICE_VERSION: ${SERVICE_VERSION:-} ports: - - "3007:3000" + - "3008:3000" environment: PORT: "3000" HOST: "0.0.0.0" diff --git a/infra/docker/service-csv/Dockerfile b/infra/docker/service-csv/Dockerfile new file mode 100644 index 00000000..897bf478 --- /dev/null +++ b/infra/docker/service-csv/Dockerfile @@ -0,0 +1,28 @@ +FROM node:24-alpine AS builder + +WORKDIR /build +COPY app/ ./ + +RUN npm ci +RUN npm run build:packages +RUN npm run -w @mdeo/service-csv build:static + +FROM node:24-alpine + +RUN apk add --no-cache curl + +WORKDIR /workspace + +COPY --from=builder /build/packages /workspace/packages +COPY --from=builder /build/node_modules /workspace/node_modules +COPY --from=builder /build/package.json /workspace/ + +WORKDIR /workspace/packages/service-csv + +ENV NODE_ENV=production +ENV PORT=3000 +ENV HOST=0.0.0.0 + +EXPOSE 3000 + +CMD ["node", "dist/index.js"] diff --git a/infra/docker/service-model-csv/Dockerfile b/infra/docker/service-model-csv/Dockerfile new file mode 100644 index 00000000..cc3293d7 --- /dev/null +++ b/infra/docker/service-model-csv/Dockerfile @@ -0,0 +1,38 @@ +# Build stage +FROM node:24-alpine AS builder + +WORKDIR /build +COPY app/ ./ + +RUN npm ci +RUN npm run build:packages +RUN npm run build:css +RUN npm run -w @mdeo/service-model-csv build:static + +# Production stage +FROM node:24-alpine + +# Install curl for healthchecks +RUN apk add --no-cache curl + +ARG SERVICE_VERSION="" + +WORKDIR /workspace + +# Copy the entire packages structure with built artifacts and dependencies +COPY --from=builder /build/packages /workspace/packages +COPY --from=builder /build/node_modules /workspace/node_modules +COPY --from=builder /build/package.json /workspace/ + +# Set working directory to service-model-csv +WORKDIR /workspace/packages/service-model-csv + +# Set environment variables +ENV NODE_ENV=production +ENV PORT=3000 +ENV HOST=0.0.0.0 +ENV SERVICE_VERSION=${SERVICE_VERSION} + +EXPOSE 3000 + +CMD ["node", "dist/index.js"] diff --git a/infra/docker/workbench/nginx.conf b/infra/docker/workbench/nginx.conf index ec68968b..f39a7997 100644 --- a/infra/docker/workbench/nginx.conf +++ b/infra/docker/workbench/nginx.conf @@ -100,6 +100,27 @@ server { add_header Cross-Origin-Embedder-Policy "require-corp" always; } + # CSV plugin + location ^~ /plugin/csv/ { + proxy_pass ${PLUGIN_CSV_SERVICE}/; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Execution results are read over a WebSocket the backend holds open to this service. + proxy_set_header Upgrade $proxy_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + add_header Cross-Origin-Resource-Policy "cross-origin" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + add_header Cross-Origin-Embedder-Policy "require-corp" always; + } + # Model-transformation plugin location ^~ /plugin/model-transformation/ { proxy_pass ${PLUGIN_MODEL_TRANSFORMATION_SERVICE}/; @@ -184,6 +205,27 @@ server { add_header Cross-Origin-Embedder-Policy "require-corp" always; } + # Model-csv plugin + location ^~ /plugin/model-csv/ { + proxy_pass ${PLUGIN_MODEL_CSV_SERVICE}/; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Execution results are read over a WebSocket the backend holds open to this service. + proxy_set_header Upgrade $proxy_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + add_header Cross-Origin-Resource-Policy "cross-origin" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + add_header Cross-Origin-Embedder-Policy "require-corp" always; + } + # Proxy API requests to backend location ^~ /api/ { proxy_pass ${BACKEND_SERVICE};