From 1f0d90059d3f40ab43884cf60bf10a3794943991 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:31:08 -0400 Subject: [PATCH 01/41] Add CSV model import: inference engine, backend endpoint, and unit tests --- infra/.DS_Store | Bin 0 -> 8196 bytes infra/docker/.DS_Store | Bin 0 -> 8196 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 infra/.DS_Store create mode 100644 infra/docker/.DS_Store diff --git a/infra/.DS_Store b/infra/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5e79992e6df7f2ec950fb3c6c5cc8bb917496f18 GIT binary patch literal 8196 zcmeHMYitx%6u#fIlo>m~v=%7HWVbG_;9?&Xq#(Go?<9Xp+=JF{CJ zsVOl&FeVybiNA=?pN$#~8iS(0OngPtkVqmSY9jve7k@ATJu`Qf*oB&y7y~*tnS17( zd(N3TXTNiA&y+ES*1XopSS@2rq|2j9B~8~E-nw^PGlW1=P84L%n9g#UF=sSm+L>## zLqV8I?}^!#T`An1O3E;FV7W)0oY&Y`CO{8;_Qy#vGMh-Dw=Q6Sg}*qoB6 zo+=H=bP8?!lSEmv7}D8dNYX8p@*K4%)!Qfcr;My^C5^pr_9&&lR&)7)b_hHKbXZ_3pVxkX3QmQsC=opalKX9J3o zcl5nF`7#Q6J9#8m)!x%~k9uyLeR|Fz8cQ_r5T(hv3nNQcu5P;P?!?xeo%c`8Q>v=v z8hJra!ElU>sdtYVnmg2+cCxyq8P;GQCD(F|gNB}yst}jDtn`RJRb9Ji@oh^~wN$(4 zC6&(@59bZ{uo4qR{){>`CMoZU{TjVwr>%J&HGf5|Q&))Meox+((`n{nd6gQMl(g98 zP=dINdbv?j9I0!V5Wr2F<#nQ>N!?~TqnljRB5xFxT~behN6v*c@;#!I%Gx6%6pX`V zt!lfdd?r5NJ-8$7xVuMntN2Q->JCxtbB5A6efJpQuV~RBivABva?fO|X1I3F&&GXq zs@m(h+M;Q?MnNyGT~w_m)oxMwmTVNmDj0-#`Ykx{#aIz@vBWBV8#_IG-oNv6EY%bh z0_vS_DqGIh(|Oj#4pO}xV^6aa>|J(>eZ;tqZ&)F6su8> z1~g&=HewStV;8#6jeW>q7$z*(a50WYFo9!u9M9lcJcsA;GG4`Ncmr=@5-0IKKES8= z9ADroe2sJX85i&iF5)t-;CG=)SS&0RmI=#+dZ9sR64nUYgzdr}p;yQVgTfGxKq!{m zf$5P6hl!uIgHX8zUjT*4>z7+(`;MLW${ja+xqUa+E3jK@<}X+nUAC&edHtp>S7FFe zTLkK#3&6+xv`X zJI^k#i|i6APzePQti+vYCS-5J4nlVaItkq=^kF~xp<@sR9wMx}DButd;|LzbV|W5j z;y5Aw1-yuta00Io+FvKUPvR}Sjd$=NPT^ym#wR}Rr|>;~z>g(J?wpQfTr5R$A(^vG z+ZrT!5&hFmZf>rOzkaRP|8Jf7`~S@yWcc=B2Eq*dWd=~*o@{R=m6FPWS_>3pvRlh7xY!E?DF`lYxk)R?wsc#HbuU9Zb~;mcX182Y zQ)9efOf=pSpNQATMvVrILD44@Z_zX)l1PY}h%Y|zf(ht<=FAdW+7~qjbWU>4fBy4d zX8wP_|C~L$j4?Flv-ONsF~(%-oN8q>Ow+iW_v@MztdtT3*)x`A2D4d)4Wvh|(+(HH z41^g7GZ1DV%s`ld8zBR9X7eV`aPA9fIEEPrGw{F60N)>?)H#g@bWBix>!88E0ubd_ z0Kd_k?g73{G@$W-jtT0Ul237aKwygCiUDCx@+em)8V~50pu(I%m@@=VMsP!c|8%m8 z`sxgELBlc3K$wB+Gr-G7VI$08IW~Ch{oNf-TTYs!C)2~0>3Vf_XCRf7&YC?(o+FpZ z2NMI%NWx2bdAB*0JHWXP$Fb7Ic`Ci%G)DTg$`;4=Ql@R>d;>FUkV?g$aap4SP<+7 zZ0?lUo}NB1%Pq8sz)BLuz|Ix}Q*J3!=jvU_?jE%_X=WTdVIDR5wD~SYw~N@lIio;P zb!8kYKV-XoT6rdAW%5?aGqze5SEjZ3ZYJkg)?UXoJ=3wflb&(dD>|AaO7^%;&TH|V z^=Vq(HTD_g%PbhJ@1K~j zl~<^h>cXyq>6&TF=o~S#-avQC%@}spwEKG~#kOZ2GL4*4j+oM6r-qD)imJs+Zd8sWDBAmSZnmmX0AFv(88nJzb+sN- zw3OW8Qi6nwTD4x$T%}`>a3D+@)%CKLRXVLy+OWh@le$UPb}L;45xH2bRPT|MWX2g9 zqF`LH)~vV6+UN2E{K0K0*V{8}*u__B)_2Nsk2{dc8GA+ug++@tS@u6zTIVL)vZm+c z{A}D;t?S*~)z)m*$WqWtsuox13B6O+z9SpWvv$7yVH_v% z0Y1cM_yS+zYkY%q_yrg6D=y+PuHX-;Tv{S6la@;>q*|#?YLM1S+oc`SUa4D3Oa0P- zh(Ib9-NDI`Nk@pEW;dat3w9G}eEOn`?AW>MUbXFpFS_sN@iM!$a>2qy(dDaa8#ip; zdKHWUyG6KOTLwPzC)JCOd>8#nOKlD>VR?dUSKqEiv{_Ute(rV460ns#Z#MDP&u8Vp+g6w~!lNW%56b z_ix!*cAi~e7uhAuMj13junKpgk?_48I|<`$XeW#((Srl%g@JyUc!&`1p@73Uf}?m0 zkK;)^h2w;pTH0J5kF1AbNggGv2qt{XFau!*{!#`|+L~x>CexWVGk({O zQh$&-Z(MIoP~U_GuEN*->p0P?e;Cp{8pz@S9TSu!)c*A!0{+&Q?(qB%&wn=XUpW8& E1mFaK5dZ)H literal 0 HcmV?d00001 From d6139d54843aefbe045a7c99be78cf5c42c56067 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:35:16 -0400 Subject: [PATCH 02/41] Remove edge label fix from CSV import branch (belongs in separate PR) --- .../diagram-server/modelTransformationGModelFactory.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/packages/language-model-transformation/src/features/diagram-server/modelTransformationGModelFactory.ts b/app/packages/language-model-transformation/src/features/diagram-server/modelTransformationGModelFactory.ts index 349e2ce1..26ed4c5b 100644 --- a/app/packages/language-model-transformation/src/features/diagram-server/modelTransformationGModelFactory.ts +++ b/app/packages/language-model-transformation/src/features/diagram-server/modelTransformationGModelFactory.ts @@ -803,7 +803,7 @@ export class ModelTransformationGModelFactory extends BaseGModelFactory Date: Fri, 10 Jul 2026 11:28:22 -0400 Subject: [PATCH 03/41] Add CSV import block syntax to model language and CSV-to-ModelData conversion --- .../language-model/src/features/csvImport.ts | 228 ++++++++++++++++++ .../language-model/src/grammar/modelRules.ts | 90 +++---- .../language-model/src/grammar/modelTypes.ts | 100 +++----- app/packages/language-model/src/index.ts | 1 + .../src/handler/modelDataHandler.ts | 75 ++++-- 5 files changed, 357 insertions(+), 137 deletions(-) create mode 100644 app/packages/language-model/src/features/csvImport.ts diff --git a/app/packages/language-model/src/features/csvImport.ts b/app/packages/language-model/src/features/csvImport.ts new file mode 100644 index 00000000..12ec6785 --- /dev/null +++ b/app/packages/language-model/src/features/csvImport.ts @@ -0,0 +1,228 @@ +import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "./modelData.js"; + +export interface CsvImportEntry { + className: string; + csvText: string; +} + +export interface CsvImportResult { + instances: ModelDataInstance[]; + links: ModelData["links"]; + warnings: string[]; +} + +/** + * Maps one or more CSV files onto a metamodel, producing model instances and links. + * Each entry maps one class to one CSV file (one row = one instance). + * Column names must match property names in the metamodel class. + * Cross-object references use a reserved `_id` column as the row identifier, + * with reference columns holding the target `_id` value. + */ +export function importCsvEntries( + entries: CsvImportEntry[], + metamodelClasses: MetamodelClassInfo[] +): CsvImportResult { + const warnings: string[] = []; + const allInstances: ModelDataInstance[] = []; + const allLinks: ModelData["links"] = []; + + const idMap = new Map(); + + for (const entry of entries) { + const classInfo = metamodelClasses.find(c => c.name === entry.className); + if (!classInfo) { + warnings.push(`Class '${entry.className}' not found in metamodel — skipping.`); + continue; + } + + const rows = parseCsv(entry.csvText); + if (rows.length < 2) { + warnings.push(`CSV for class '${entry.className}' has no data rows — skipping.`); + continue; + } + + const header = rows[0]; + const dataRows = rows.slice(1); + const idColIndex = header.indexOf("_id"); + + const propertyNames = new Set(classInfo.properties.map(p => p.name)); + const unknownCols = header.filter(h => h !== "_id" && !propertyNames.has(h)); + if (unknownCols.length > 0) { + warnings.push(`CSV for '${entry.className}' has unknown columns: ${unknownCols.join(", ")} — they will be ignored.`); + } + + dataRows.forEach((row, rowIndex) => { + const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); + const instanceName = `${entry.className}_${rowIndex}`; + + if (idColIndex >= 0) { + const idValue = normalizedRow[idColIndex]; + if (idValue) { + idMap.set(`${entry.className}:${idValue}`, instanceName); + } + } + + const properties: Record = {}; + header.forEach((colName, colIndex) => { + if (colName === "_id") return; + const prop = classInfo.properties.find(p => p.name === colName); + if (!prop) return; + properties[colName] = convertCellValue(normalizedRow[colIndex], prop); + }); + + allInstances.push({ + name: instanceName, + className: entry.className, + properties + }); + }); + } + + for (const entry of entries) { + const classInfo = metamodelClasses.find(c => c.name === entry.className); + if (!classInfo) continue; + + const rows = parseCsv(entry.csvText); + if (rows.length < 2) continue; + + const header = rows[0]; + const dataRows = rows.slice(1); + + const refCols = header.filter(colName => { + const prop = classInfo.properties.find(p => p.name === colName); + return prop?.isReference; + }); + + if (refCols.length === 0) continue; + + dataRows.forEach((row, rowIndex) => { + const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); + const sourceInstanceName = `${entry.className}_${rowIndex}`; + + refCols.forEach(colName => { + const prop = classInfo.properties.find(p => p.name === colName)!; + const rawValue = normalizedRow[header.indexOf(colName)]; + if (!rawValue) return; + + const targetIds = rawValue.split(";").map(s => s.trim()).filter(Boolean); + targetIds.forEach(targetId => { + const targetInstanceName = idMap.get(`${prop.referencedClass}:${targetId}`); + if (!targetInstanceName) { + warnings.push(`Reference '${targetId}' in column '${colName}' of '${entry.className}' row ${rowIndex + 2} could not be resolved.`); + return; + } + allLinks.push({ + sourceName: sourceInstanceName, + sourceProperty: colName, + targetName: targetInstanceName, + targetProperty: null + }); + }); + }); + }); + } + + return { instances: allInstances, links: allLinks, warnings }; +} + +export interface MetamodelPropertyInfo { + name: string; + type: "string" | "int" | "long" | "double" | "float" | "boolean" | "enum" | "reference"; + enumEntries?: string[]; + isReference?: boolean; + referencedClass?: string; +} + +export interface MetamodelClassInfo { + name: string; + properties: MetamodelPropertyInfo[]; +} + +function convertCellValue(rawValue: string, prop: MetamodelPropertyInfo): ModelDataPropertyValue { + if (!rawValue || rawValue.trim() === "") return null; + const v = rawValue.trim(); + switch (prop.type) { + case "int": + case "long": { + const n = parseInt(v, 10); + return isNaN(n) ? v : n; + } + case "double": + case "float": { + const n = parseFloat(v); + return isNaN(n) ? v : n; + } + case "boolean": + return v.toLowerCase() === "true"; + case "enum": + return { enum: v }; + default: + return v; + } +} + +function normalizeRow(row: string[], expectedLength: number, rowNumber: number, warnings: string[]): string[] { + if (row.length === expectedLength) return row; + if (row.length < expectedLength) { + warnings.push(`Row ${rowNumber} has fewer columns than the header; missing values treated as blank.`); + return [...row, ...Array(expectedLength - row.length).fill("")]; + } + warnings.push(`Row ${rowNumber} has more columns than the header; extra values ignored.`); + return row.slice(0, expectedLength); +} + +function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let currentRow: string[] = []; + let currentField = ""; + let inQuotes = false; + let i = 0; + + const endField = () => { + currentRow.push(currentField); + currentField = ""; + }; + + const endRow = () => { + endField(); + if (!(currentRow.length === 1 && currentRow[0].trim() === "")) { + rows.push(currentRow); + } + currentRow = []; + }; + + while (i < text.length) { + const c = text[i]; + if (inQuotes) { + if (c === '"') { + if (i + 1 < text.length && text[i + 1] === '"') { + currentField += '"'; + i++; + } else { + inQuotes = false; + } + } else { + currentField += c; + } + } else { + if (c === '"') { + inQuotes = true; + } else if (c === ',') { + endField(); + } else if (c === '\r') { + // skip + } else if (c === '\n') { + endRow(); + } else { + currentField += c; + } + } + i++; + } + + if (currentField !== "" || currentRow.length > 0) { + endRow(); + } + + return rows; +} diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index cce0b9c9..a2a0c52b 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -27,47 +27,29 @@ import { Model, MetamodelFileImport, LinkEnd, - Link + Link, + CsvClassImport, + CsvImportBlock } from "./modelTypes.js"; -/** - * Boolean literal rule. - * Matches "true" or "false". - */ export const BOOLEAN = createRule("BOOLEAN") .returns(Boolean) .as(() => [or("true", "false")]); -/** - * Simple value rule. - * Matches string, number, or boolean literals. - */ export const SimpleValueRule = createRule("SimpleValueRule") .returns(SimpleValue) .as(({ set }) => [ or(set("stringValue", STRING), set("numberValue", FLOAT), set("numberValue", INT), set("booleanValue", BOOLEAN)) ]); -/** - * Enum value rule. - * Matches an enum entry using EnumName.Entry syntax. - */ export const EnumValueRule = createRule("EnumValueRule") .returns(EnumValue) .as(({ set }) => [set("enumRef", ref(Enum, ID)), ".", set("value", ref(EnumEntry, ID))]); -/** - * Single value rule. - * Matches either a simple value or an enum value. - */ export const SingleValueRule = createRule("SingleValueRule") .returns(SingleValue) .as(() => [or(SimpleValueRule, EnumValueRule)]); -/** - * List value rule. - * Matches values in square brackets with comma separation. - */ export const ListValueRule = createRule("ListValueRule") .returns(ListValue) .as(({ add }) => [ @@ -76,26 +58,14 @@ export const ListValueRule = createRule("ListValueRule") "]" ]); -/** - * Literal value rule. - * Matches any value type: simple, enum, or list. - */ export const LiteralValueRule = createRule("LiteralValueRule") .returns(LiteralValue) .as(() => [or(ListValueRule, SimpleValueRule, EnumValueRule)]); -/** - * Property assignment rule. - * Matches property assignments like "a = 100". - */ export const PropertyAssignmentRule = createRule("PropertyAssignmentRule") .returns(PropertyAssignment) .as(({ set }) => [set("name", ref(Property, ID)), "=", set("value", LiteralValueRule)]); -/** - * Object instance rule. - * Matches object definitions like "test : ClassName { ... }". - */ export const ObjectInstanceRule = createRule("ObjectInstanceRule") .returns(ObjectInstance) .as(({ set, add }) => [ @@ -107,10 +77,6 @@ export const ObjectInstanceRule = createRule("ObjectInstanceRule") "}" ]); -/** - * Link end rule. - * Matches an object reference with optional property specification. - */ export const LinkEndRule = createRule("LinkEndRule") .returns(LinkEnd) .as(({ set }) => [ @@ -118,35 +84,59 @@ export const LinkEndRule = createRule("LinkEndRule") optional(group(".", set("property", ref(AssociationEnd, ID)))) ]); -/** - * Link rule. - * Matches links between object instances: objectId1[.property] -- objectId2[.property] - */ export const LinkRule = createRule("LinkRule") .returns(Link) .as(({ set }) => [set("source", LinkEndRule), "--", set("target", LinkEndRule)]); -/** - * Metamodel file import rule. - * Matches "using" statements for importing metamodel files. - */ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") .returns(MetamodelFileImport) .as(({ set }) => ["using", set("file", STRING)]); +/** + * CSV class import rule. + * Format: ClassName from "path/to/file.csv" + */ +export const CsvClassImportRule = createRule("CsvClassImportRule") + .returns(CsvClassImport) + .as(({ set }) => [ + set("class", ref(Class, ID)), + "from", + set("file", STRING) + ]); + +/** + * CSV import block rule. + * Format: + * import CSV { + * ClassName from "file.csv" + * } + */ +export const CsvImportBlockRule = createRule("CsvImportBlockRule") + .returns(CsvImportBlock) + .as(({ add }) => [ + "import", + "CSV", + "{", + many(or(add("imports", CsvClassImportRule), NEWLINE)), + "}" + ]); + /** * Model root rule. - * Matches the complete model file structure. + * A model file starts with a metamodel import, then either: + * - A CSV import block (no hand-authored objects allowed), or + * - Hand-authored object instances and links */ export const ModelRule = createRule("ModelRule") .returns(Model) .as(({ add, set }) => [ many(NEWLINE), set("import", MetamodelFileImportRule), - many(or(add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) + many(NEWLINE), + or( + set("csvImport", CsvImportBlockRule), + many(or(add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) + ) ]); -/** - * Additional terminals for the Model language. - */ export const ModelTerminals = [WS, HIDDEN_NEWLINE, ML_COMMENT, SL_COMMENT]; diff --git a/app/packages/language-model/src/grammar/modelTypes.ts b/app/packages/language-model/src/grammar/modelTypes.ts index b0d7c00a..70083c10 100644 --- a/app/packages/language-model/src/grammar/modelTypes.ts +++ b/app/packages/language-model/src/grammar/modelTypes.ts @@ -2,152 +2,106 @@ import { createInterface, createType, Optional, Ref, type ASTType, type BaseType import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import type { AstNode } from "langium"; -/** - * Simple value for property assignments. - * Supports string, number, and boolean literals. - */ export const SimpleValue = createInterface("SimpleValue").attrs({ stringValue: Optional(String), numberValue: Optional(Number), booleanValue: Optional(Boolean) }); -/** - * Type representing a SimpleValue AST node. - */ export type SimpleValueType = ASTType; -/** - * Enum value for property assignments. - * References an enum and one of its entries from the metamodel. - * Written as EnumName.Entry in the model language. - */ export const EnumValue = createInterface("EnumValue").attrs({ enumRef: Ref(() => Enum), value: Ref(() => EnumEntry) }); -/** - * Type representing an EnumValue AST node. - */ export type EnumValueType = ASTType; -/** - * Single value type union. - * Represents either a simple value or an enum value. - */ export const SingleValue: BaseType = createType("SingleValue").types(SimpleValue, EnumValue); -/** - * Type representing a SingleValue AST node. - */ export type SingleValueType = ASTType; -/** - * List value for property assignments with multiplicity > 1. - * Contains a comma-separated list of values in square brackets. - */ export const ListValue = createInterface("ListValue").attrs({ values: [SingleValue] }); -/** - * Type representing a ListValue AST node. - */ export type ListValueType = ASTType; -/** - * Literal value type union for property assignments. - * Can be a single value (simple or enum) or a list value. - */ export const LiteralValue: BaseType = createType("LiteralValue").types(SimpleValue, EnumValue, ListValue); -/** - * Type representing a LiteralValue AST node. - */ export type LiteralValueType = ASTType; -/** - * Property assignment in an object instance. - * References a property from the class and assigns it a value. - */ export const PropertyAssignment = createInterface("PropertyAssignment").attrs({ name: Ref(() => Property), value: LiteralValue }); -/** - * Type representing a PropertyAssignment AST node. - */ export type PropertyAssignmentType = ASTType; -/** - * Object instance definition. - * Represents an instance of a class from the metamodel. - */ export const ObjectInstance = createInterface("ObjectInstance").attrs({ name: String, class: Ref(() => Class), properties: [PropertyAssignment] }); -/** - * Type representing an ObjectInstance AST node. - */ export type ObjectInstanceType = ASTType; -/** - * Link end representing one side of a link. - * References an object instance with an optional property specification. - */ export const LinkEnd = createInterface("LinkEnd").attrs({ object: Ref(() => ObjectInstance), property: Optional(Ref(() => AssociationEnd)) }); -/** - * Type representing a LinkEnd AST node. - */ export type LinkEndType = ASTType; -/** - * Link definition between two object instances. - * Format: objectId1[.property] -- objectId2[.property] - */ export const Link = createInterface("Link").attrs({ source: LinkEnd, target: LinkEnd }); -/** - * Type representing a Link AST node. - */ export type LinkType = ASTType; +export const MetamodelFileImport = createInterface("MetamodelFileImport").attrs({ + file: String +}); + +export type MetamodelFileImportType = ASTType; + /** - * Metamodel file import. - * Simple import statement for referencing metamodel files. + * A single CSV class import entry. + * Maps a metamodel class to a CSV file. + * Format: ClassName from "path/to/file.csv" */ -export const MetamodelFileImport = createInterface("MetamodelFileImport").attrs({ +export const CsvClassImport = createInterface("CsvClassImport").attrs({ + class: Ref(() => Class), file: String }); +export type CsvClassImportType = ASTType; + /** - * Type representing MetamodelFileImport AST node. + * A CSV import block at the top of a model file. + * When present, forbids hand-authored object instances. + * Format: + * import CSV { + * ClassName from "file.csv" + * } */ -export type MetamodelFileImportType = ASTType; +export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ + imports: [CsvClassImport] +}); + +export type CsvImportBlockType = ASTType; /** * Model root interface. - * Contains metamodel imports, object instances, and links. + * Contains metamodel import, optional CSV import block, optional hand-authored objects and links. + * Either csvImport or objects/links may be present, but not both. */ export const Model = createInterface("Model").attrs({ import: MetamodelFileImport, + csvImport: Optional(CsvImportBlock), objects: [ObjectInstance], links: [Link] }); -/** - * Model AST type. - */ export type ModelType = ASTType; diff --git a/app/packages/language-model/src/index.ts b/app/packages/language-model/src/index.ts index 4b4ba414..5a92df5e 100644 --- a/app/packages/language-model/src/index.ts +++ b/app/packages/language-model/src/index.ts @@ -1,4 +1,5 @@ export * from "./action-handlers/newFileActionHandler.js"; +export * from "./features/csvImport.js"; export * from "./action-handlers/saveGeneratedModelActionHandler.js"; export * from "./features/diagram-server/generated/generatedModelCreateEdgeSchemaResolver.js"; export * from "./features/diagram-server/generated/generatedModelDiagramConfiguration.js"; diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index 3bbc6ff8..226a86ce 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -1,20 +1,10 @@ import { type ModelData, type ModelServices, ModelDataConverter, Model } from "@mdeo/language-model"; +import { type MetamodelClassInfo, type MetamodelPropertyInfo, importCsvEntries } from "@mdeo/language-model"; import { hasErrors, type FileDataHandler } from "@mdeo/service-common"; +import { resolveRelativePath } from "@mdeo/language-shared"; -/** - * Key for the model data handler. - */ export const MODEL_DATA_HANDLER_KEY = "model-data"; -/** - * Handler for computing the ModelData representation of a model file. - * Converts the language AST into a lightweight format suitable for serialization. - * - * Returns null if the document has errors to ensure only valid models are returned. - * - * @param context The file data context with path, content, and services - * @returns Promise resolving to the file data result with ModelData or null - */ export const modelDataHandler: FileDataHandler = async (context) => { const { instance, fileInfo, serverApi } = context; @@ -41,8 +31,65 @@ export const modelDataHandler: FileDataHandler throw new Error("Document root is not a Model"); } - const converter = new ModelDataConverter(reflection); - const modelData = converter.convertModel(model); + if (model.csvImport == undefined) { + const converter = new ModelDataConverter(reflection); + const modelData = converter.convertModel(model); + return { + data: modelData, + ...serverApi.getTrackedRequests() + }; + } + + const metamodelPath = model.import?.file; + if (!metamodelPath) { + return { data: null, ...serverApi.getTrackedRequests() }; + } + + const metamodelUri = resolveRelativePath(document, metamodelPath); + const metamodelDoc = instance.services.shared.workspace.LangiumDocuments.getDocument(metamodelUri); + if (!metamodelDoc) { + return { data: null, ...serverApi.getTrackedRequests() }; + } + + const metamodelAst = metamodelDoc.parseResult?.value as any; + const metamodelClasses: MetamodelClassInfo[] = (metamodelAst?.elements ?? []) + .filter((el: any) => el.$type === "Class") + .map((cls: any): MetamodelClassInfo => ({ + name: cls.name, + properties: (cls.properties ?? []).map((prop: any): MetamodelPropertyInfo => { + const primitiveType = prop.type?.name; + const enumRef = prop.type?.enum?.ref; + const isReference = prop.$type === "AssociationEnd"; + return { + name: prop.name, + type: isReference ? "reference" + : enumRef ? "enum" + : (primitiveType ?? "string") as MetamodelPropertyInfo["type"], + enumEntries: enumRef ? enumRef.entries?.map((e: any) => e.name) : undefined, + isReference, + referencedClass: isReference ? prop.class?.ref?.name : undefined + }; + }) + })); + + const csvEntries = await Promise.all( + model.csvImport.imports.map(async (entry: any) => { + const csvPath: string = resolveRelativePath(document, entry.file).toString(); + const { content } = await serverApi.readFile(csvPath); + return { + className: entry.class?.ref?.name ?? "", + csvText: content + }; + }) + ); + + const result = importCsvEntries(csvEntries, metamodelClasses); + + const modelData: ModelData = { + metamodelPath, + instances: result.instances, + links: result.links + }; return { data: modelData, From 4209220e3b7873553494c46df632761053e09fb4 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:57:53 -0400 Subject: [PATCH 04/41] Fix CSV import: use fileDataService to get metamodel data instead of parsing raw file --- .../mdeo/backend/service/CsvImportService.kt | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt index 06109396..8257b8ba 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt @@ -12,7 +12,6 @@ import java.util.UUID @Serializable data class CsvImportResult( - val metamodelPath: String, val modelPath: String, val instanceCount: Int, val warnings: List @@ -26,29 +25,38 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ projectId: UUID, csvText: String, basePath: String, - className: String + className: String, + metamodelPath: String ): ApiResult { val normalizedBasePath = basePath.trim().trim('/') - val metamodelPath = "$normalizedBasePath.metamodel" - val modelPath = "$normalizedBasePath.model" + if (normalizedBasePath.isBlank()) { + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Invalid path parameter") + } + val modelPath = "$normalizedBasePath.m" + + val metamodelResult = fileService.readFile(projectId, metamodelPath) + if (metamodelResult is ApiResult.Failure) { + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not read metamodel at '$metamodelPath'") + } + val metamodelBytes = (metamodelResult as ApiResult.Success).value + val metamodel = try { + json.decodeFromString(metamodelBytes.decodeToString()) + } catch (e: Exception) { + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not parse metamodel: ${e.message}") + } - val inference = try { - CsvModelInference.inferFromCsv( + val importResult = try { + CsvModelInference.importFromCsv( csvText = csvText, className = className, + metamodel = metamodel, metamodelPath = metamodelPath ) } catch (e: IllegalArgumentException) { return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, e.message ?: "Invalid CSV") } - val metamodelBytes = json.encodeToString(inference.metamodel).toByteArray(Charsets.UTF_8) - val modelBytes = json.encodeToString(inference.model).toByteArray(Charsets.UTF_8) - - when (val result = fileService.writeFile(projectId, metamodelPath, metamodelBytes, create = true, overwrite = true)) { - is ApiResult.Failure -> return ApiResult.Failure(result.error) - is ApiResult.Success -> {} - } + val modelBytes = json.encodeToString(importResult.model).toByteArray(Charsets.UTF_8) when (val result = fileService.writeFile(projectId, modelPath, modelBytes, create = true, overwrite = true)) { is ApiResult.Failure -> return ApiResult.Failure(result.error) @@ -57,10 +65,9 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ return success( CsvImportResult( - metamodelPath = metamodelPath, modelPath = modelPath, - instanceCount = inference.model.instances.size, - warnings = inference.warnings + instanceCount = importResult.model.instances.size, + warnings = importResult.warnings ) ) } From 1fe2d53d07b6f75e17eb55403a33483c4995c2cb Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:06:11 -0400 Subject: [PATCH 05/41] Fix CSV import: use fileDataService to get computed metamodel data --- .../mdeo/backend/service/CsvImportService.kt | 191 +++++++++++++++--- 1 file changed, 168 insertions(+), 23 deletions(-) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt index 8257b8ba..40d79816 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt @@ -4,10 +4,8 @@ import com.mdeo.common.model.ApiResult import com.mdeo.common.model.ErrorCodes import com.mdeo.common.model.commonFailure import com.mdeo.common.model.success -import com.mdeo.metamodel.csv.CsvModelInference import kotlinx.serialization.Serializable -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json +import kotlinx.serialization.json.* import java.util.UUID @Serializable @@ -19,9 +17,9 @@ data class CsvImportResult( class CsvImportService(services: InjectedServices) : BaseService(), InjectedServices by services { - private val json = Json { prettyPrint = true } + private val json = Json { ignoreUnknownKeys = true; prettyPrint = true } - fun importCsv( + suspend fun importCsv( projectId: UUID, csvText: String, basePath: String, @@ -34,29 +32,75 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ } val modelPath = "$normalizedBasePath.m" - val metamodelResult = fileService.readFile(projectId, metamodelPath) - if (metamodelResult is ApiResult.Failure) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not read metamodel at '$metamodelPath'") + val metamodelDataResult = fileDataService.getFileData(projectId, metamodelPath, null, "metamodel") + if (metamodelDataResult is ApiResult.Failure) { + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not compute metamodel data for '$metamodelPath'") } - val metamodelBytes = (metamodelResult as ApiResult.Success).value - val metamodel = try { - json.decodeFromString(metamodelBytes.decodeToString()) + val metamodelJson = (metamodelDataResult as ApiResult.Success).value.data + ?: return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Metamodel '$metamodelPath' has errors") + + val metamodelData = try { + json.decodeFromJsonElement(metamodelJson) } catch (e: Exception) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not parse metamodel: ${e.message}") + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not parse metamodel data: ${e.message}") + } + + val targetClass = metamodelData.classes.find { it.name == className } + ?: return commonFailure( + ErrorCodes.CSV_IMPORT_FAILED, + "Class '$className' not found in metamodel. Available: ${metamodelData.classes.map { it.name }}" + ) + + val warnings = mutableListOf() + + val rows = parseCsv(csvText) + if (rows.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV has no header row") + val header = rows.first() + if (header.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV header row is empty") + val dataRows = rows.drop(1) + if (dataRows.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV has no data rows") + + val propertyNames = targetClass.properties.map { it.name }.toSet() + header.forEach { col -> + if (col != "_id" && col !in propertyNames) { + warnings.add("Column '$col' does not match any property in class '$className' — it will be ignored.") + } } - val importResult = try { - CsvModelInference.importFromCsv( - csvText = csvText, - className = className, - metamodel = metamodel, - metamodelPath = metamodelPath + val missingRequired = targetClass.properties.filter { prop -> + prop.multiplicity.lower > 0 && header.none { it == prop.name } + } + if (missingRequired.isNotEmpty()) { + return commonFailure( + ErrorCodes.CSV_IMPORT_FAILED, + "CSV is missing required properties: ${missingRequired.map { it.name }}" ) - } catch (e: IllegalArgumentException) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, e.message ?: "Invalid CSV") } - val modelBytes = json.encodeToString(importResult.model).toByteArray(Charsets.UTF_8) + val instances = dataRows.mapIndexed { rowIndex, row -> + val normalizedRow = normalizeRow(row, header.size, rowIndex + 2, warnings) + val properties = buildJsonObject { + header.forEachIndexed { colIndex, colName -> + if (colName == "_id") return@forEachIndexed + val prop = targetClass.properties.find { it.name == colName } ?: return@forEachIndexed + val rawValue = normalizedRow[colIndex] + put(colName, convertCell(rawValue, prop, metamodelData)) + } + } + buildJsonObject { + put("name", "${className}_$rowIndex") + put("className", className) + put("properties", properties) + } + } + + val modelContent = buildJsonObject { + put("metamodelPath", metamodelPath) + put("instances", JsonArray(instances)) + put("links", JsonArray(emptyList())) + } + + val modelBytes = json.encodeToString(modelContent).toByteArray(Charsets.UTF_8) when (val result = fileService.writeFile(projectId, modelPath, modelBytes, create = true, overwrite = true)) { is ApiResult.Failure -> return ApiResult.Failure(result.error) @@ -66,9 +110,110 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ return success( CsvImportResult( modelPath = modelPath, - instanceCount = importResult.model.instances.size, - warnings = importResult.warnings + instanceCount = instances.size, + warnings = warnings ) ) } + + private fun convertCell(rawValue: String, prop: PropertyData, metamodel: MetamodelAstData): JsonElement { + if (rawValue.isBlank()) return JsonNull + return when { + prop.enumType != null -> JsonPrimitive(rawValue) + prop.primitiveType == "int" || prop.primitiveType == "long" -> + rawValue.toLongOrNull()?.let { JsonPrimitive(it) } ?: JsonPrimitive(rawValue) + prop.primitiveType == "double" || prop.primitiveType == "float" -> + rawValue.toDoubleOrNull()?.let { JsonPrimitive(it) } ?: JsonPrimitive(rawValue) + prop.primitiveType == "boolean" -> + JsonPrimitive(rawValue.equals("true", ignoreCase = true)) + else -> JsonPrimitive(rawValue) + } + } + + private fun normalizeRow(row: List, expectedSize: Int, rowNumber: Int, warnings: MutableList): List { + return when { + row.size == expectedSize -> row + row.size < expectedSize -> { + warnings.add("Row $rowNumber has fewer columns than the header; missing values treated as blank.") + row + List(expectedSize - row.size) { "" } + } + else -> { + warnings.add("Row $rowNumber has more columns than the header; extra values ignored.") + row.take(expectedSize) + } + } + } + + private fun parseCsv(text: String): List> { + val rows = mutableListOf>() + var currentRow = mutableListOf() + val currentField = StringBuilder() + var inQuotes = false + var i = 0 + + fun endField() { currentRow.add(currentField.toString()); currentField.clear() } + fun endRow() { + endField() + if (!(currentRow.size == 1 && currentRow[0].isBlank())) rows.add(currentRow) + currentRow = mutableListOf() + } + + while (i < text.length) { + val c = text[i] + if (inQuotes) { + when (c) { + '"' -> if (i + 1 < text.length && text[i + 1] == '"') { currentField.append('"'); i++ } + else inQuotes = false + else -> currentField.append(c) + } + } else { + when (c) { + '"' -> inQuotes = true + ',' -> endField() + '\r' -> {} + '\n' -> endRow() + else -> currentField.append(c) + } + } + i++ + } + if (currentField.isNotEmpty() || currentRow.isNotEmpty()) endRow() + return rows + } + + @Serializable + private data class MultiplicityData(val lower: Int, val upper: Int) + + @Serializable + private data class PropertyData( + val name: String, + val enumType: String? = null, + val primitiveType: String? = null, + val multiplicity: MultiplicityData + ) + + @Serializable + private data class ClassData( + val name: String, + val isAbstract: Boolean, + val properties: List + ) + + @Serializable + private data class EnumData(val name: String, val entries: List) + + @Serializable + private data class AssociationEndData(val className: String, val name: String? = null, val multiplicity: MultiplicityData) + + @Serializable + private data class AssociationData(val source: AssociationEndData, val operator: String, val target: AssociationEndData) + + @Serializable + private data class MetamodelAstData( + val path: String, + val classes: List, + val enums: List, + val associations: List, + val importedMetamodelPaths: List = emptyList() + ) } From f3b7602df3b893694e89db7d5150dd29eccf55a8 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:11:51 -0400 Subject: [PATCH 06/41] Fix compile error: handle JsonNull explicitly instead of elvis on non-nullable --- .../main/kotlin/com/mdeo/backend/service/CsvImportService.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt index 40d79816..21e123ee 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt @@ -37,7 +37,9 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not compute metamodel data for '$metamodelPath'") } val metamodelJson = (metamodelDataResult as ApiResult.Success).value.data - ?: return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Metamodel '$metamodelPath' has errors") + if (metamodelJson == kotlinx.serialization.json.JsonNull) { + return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Metamodel '$metamodelPath' has errors or could not be parsed") + } val metamodelData = try { json.decodeFromJsonElement(metamodelJson) From eb6cbbbe8bfaf39a07113d51a3046bf05d53f8fa Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 13:19:02 -0400 Subject: [PATCH 07/41] Write model file in DSL text format instead of JSON --- .../mdeo/backend/service/CsvImportService.kt | 52 +++++++++---------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt index 21e123ee..cd95e3f5 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt @@ -79,30 +79,27 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ ) } - val instances = dataRows.mapIndexed { rowIndex, row -> + val dslLines = StringBuilder() + dslLines.appendLine("""using "./$metamodelPath"""") + dslLines.appendLine() + + dataRows.forEachIndexed { rowIndex, row -> val normalizedRow = normalizeRow(row, header.size, rowIndex + 2, warnings) - val properties = buildJsonObject { - header.forEachIndexed { colIndex, colName -> - if (colName == "_id") return@forEachIndexed - val prop = targetClass.properties.find { it.name == colName } ?: return@forEachIndexed - val rawValue = normalizedRow[colIndex] - put(colName, convertCell(rawValue, prop, metamodelData)) - } - } - buildJsonObject { - put("name", "${className}_$rowIndex") - put("className", className) - put("properties", properties) + val instanceName = "${className}_$rowIndex" + dslLines.appendLine("$instanceName : $className {") + header.forEachIndexed { colIndex, colName -> + if (colName == "_id") return@forEachIndexed + val prop = targetClass.properties.find { it.name == colName } ?: return@forEachIndexed + val rawValue = normalizedRow[colIndex] + if (rawValue.isBlank()) return@forEachIndexed + val formattedValue = formatDslValue(rawValue, prop) + dslLines.appendLine(" $colName = $formattedValue") } + dslLines.appendLine("}") + dslLines.appendLine() } - val modelContent = buildJsonObject { - put("metamodelPath", metamodelPath) - put("instances", JsonArray(instances)) - put("links", JsonArray(emptyList())) - } - - val modelBytes = json.encodeToString(modelContent).toByteArray(Charsets.UTF_8) + val modelBytes = dslLines.toString().toByteArray(Charsets.UTF_8) when (val result = fileService.writeFile(projectId, modelPath, modelBytes, create = true, overwrite = true)) { is ApiResult.Failure -> return ApiResult.Failure(result.error) @@ -112,23 +109,22 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ return success( CsvImportResult( modelPath = modelPath, - instanceCount = instances.size, + instanceCount = dataRows.size, warnings = warnings ) ) } - private fun convertCell(rawValue: String, prop: PropertyData, metamodel: MetamodelAstData): JsonElement { - if (rawValue.isBlank()) return JsonNull + private fun formatDslValue(rawValue: String, prop: PropertyData): String { return when { - prop.enumType != null -> JsonPrimitive(rawValue) + prop.enumType != null -> rawValue prop.primitiveType == "int" || prop.primitiveType == "long" -> - rawValue.toLongOrNull()?.let { JsonPrimitive(it) } ?: JsonPrimitive(rawValue) + rawValue.toLongOrNull()?.toString() ?: "\"$rawValue\"" prop.primitiveType == "double" || prop.primitiveType == "float" -> - rawValue.toDoubleOrNull()?.let { JsonPrimitive(it) } ?: JsonPrimitive(rawValue) + rawValue.toDoubleOrNull()?.toString() ?: "\"$rawValue\"" prop.primitiveType == "boolean" -> - JsonPrimitive(rawValue.equals("true", ignoreCase = true)) - else -> JsonPrimitive(rawValue) + if (rawValue.equals("true", ignoreCase = true)) "true" else "false" + else -> "\"${rawValue.replace("\\", "\\\\").replace("\"", "\\\"")}\"" } } From 927ec8b1e1b9d55e42948773d10f0cfdb60a0593 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:26:40 -0400 Subject: [PATCH 08/41] Fix relative metamodel path in generated model DSL --- .../kotlin/com/mdeo/backend/service/CsvImportService.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt index cd95e3f5..dfeddd3a 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt @@ -79,8 +79,12 @@ class CsvImportService(services: InjectedServices) : BaseService(), InjectedServ ) } + val modelFileDepth = normalizedBasePath.count { it == '/' } + val relativePrefix = "../".repeat(modelFileDepth) + val relativeMetamodelPath = "$relativePrefix$metamodelPath" + val dslLines = StringBuilder() - dslLines.appendLine("""using "./$metamodelPath"""") + dslLines.appendLine("""using "$relativeMetamodelPath"""") dslLines.appendLine() dataRows.forEachIndexed { rowIndex, row -> From 30edfa940e218574f6c9dfc7d008367d7a7de4d6 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:07:06 -0400 Subject: [PATCH 09/41] Add language-csv and service-csv packages for CSV file support in workbench --- app/packages/language-csv/package.json | 16 +++++ app/packages/language-csv/src/csvPlugin.ts | 39 +++++++++++ .../language-csv/src/grammar/csvRules.ts | 10 +++ .../language-csv/src/grammar/csvTypes.ts | 7 ++ app/packages/language-csv/src/index.ts | 3 + app/packages/language-csv/tsconfig.json | 13 ++++ app/packages/service-csv/package.json | 26 +++++++ app/packages/service-csv/src/index.ts | 67 +++++++++++++++++++ .../service-csv/src/served/language.ts | 3 + app/packages/service-csv/tsconfig.json | 16 +++++ app/packages/service-csv/vite.config.ts | 16 +++++ app/tsconfig.build.json | 6 ++ infra/docker-compose-dev.yaml | 10 +++ infra/docker/service-csv/Dockerfile | 25 +++++++ infra/docker/workbench/nginx.conf | 6 ++ 15 files changed, 263 insertions(+) create mode 100644 app/packages/language-csv/package.json create mode 100644 app/packages/language-csv/src/csvPlugin.ts create mode 100644 app/packages/language-csv/src/grammar/csvRules.ts create mode 100644 app/packages/language-csv/src/grammar/csvTypes.ts create mode 100644 app/packages/language-csv/src/index.ts create mode 100644 app/packages/language-csv/tsconfig.json create mode 100644 app/packages/service-csv/package.json create mode 100644 app/packages/service-csv/src/index.ts create mode 100644 app/packages/service-csv/src/served/language.ts create mode 100644 app/packages/service-csv/tsconfig.json create mode 100644 app/packages/service-csv/vite.config.ts create mode 100644 infra/docker/service-csv/Dockerfile diff --git a/app/packages/language-csv/package.json b/app/packages/language-csv/package.json new file mode 100644 index 00000000..25923f33 --- /dev/null +++ b/app/packages/language-csv/package.json @@ -0,0 +1,16 @@ +{ + "name": "@mdeo/language-csv", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist", "src"], + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/language-csv/src/csvPlugin.ts b/app/packages/language-csv/src/csvPlugin.ts new file mode 100644 index 00000000..9b18a254 --- /dev/null +++ b/app/packages/language-csv/src/csvPlugin.ts @@ -0,0 +1,39 @@ +import { + type LangiumLanguagePlugin, + type LangiumLanguagePluginProvider, + type LanguageServices, + type ExternalReferenceAdditionalServices +} from "@mdeo/language-common"; +import { + DefaultAstSerializer, + SerializerFormatter, + registerDefaultTokenSerializers, + DefaultExternalReferenceCollector +} from "@mdeo/language-shared"; +import type { ServerContributionPlugin } from "@mdeo/plugin"; +import { CsvFileRule, CsvTerminals } from "./grammar/csvRules.js"; + +function createCsvPlugin(): LangiumLanguagePlugin { + return { + rootRule: CsvFileRule, + additionalTerminals: CsvTerminals, + module: { + references: { + ExternalReferenceCollector: () => new DefaultExternalReferenceCollector() + }, + lsp: { + Formatter: (services: LanguageServices & ExternalReferenceAdditionalServices) => new SerializerFormatter(services) + }, + AstSerializer: (services: LanguageServices & ExternalReferenceAdditionalServices) => new DefaultAstSerializer(services) + }, + postCreate(services: LanguageServices & ExternalReferenceAdditionalServices) { + registerDefaultTokenSerializers(services); + } + }; +} + +export const csvPluginProvider: LangiumLanguagePluginProvider = { + create(_contributionPlugins: ServerContributionPlugin[], _languageJsUrl?: string): LangiumLanguagePlugin { + return createCsvPlugin(); + } +}; diff --git a/app/packages/language-csv/src/grammar/csvRules.ts b/app/packages/language-csv/src/grammar/csvRules.ts new file mode 100644 index 00000000..f60230bd --- /dev/null +++ b/app/packages/language-csv/src/grammar/csvRules.ts @@ -0,0 +1,10 @@ +import { createTerminal, createRule, WS } from "@mdeo/language-common"; +import { CsvFile } from "./csvTypes.js"; + +export const ANY_TEXT = createTerminal("ANY_TEXT").as(/[\s\S]+/); + +export const CsvFileRule = createRule("CsvFileRule") + .returns(CsvFile) + .as(({ set }) => [set("content", ANY_TEXT)]); + +export const CsvTerminals = [WS, ANY_TEXT]; diff --git a/app/packages/language-csv/src/grammar/csvTypes.ts b/app/packages/language-csv/src/grammar/csvTypes.ts new file mode 100644 index 00000000..4f2a38af --- /dev/null +++ b/app/packages/language-csv/src/grammar/csvTypes.ts @@ -0,0 +1,7 @@ +import { createInterface, type ASTType } from "@mdeo/language-common"; + +export const CsvFile = createInterface("CsvFile").attrs({ + content: String +}); + +export type CsvFileType = ASTType; diff --git a/app/packages/language-csv/src/index.ts b/app/packages/language-csv/src/index.ts new file mode 100644 index 00000000..2b1a4096 --- /dev/null +++ b/app/packages/language-csv/src/index.ts @@ -0,0 +1,3 @@ +export * from "./grammar/csvTypes.js"; +export * from "./grammar/csvRules.js"; +export * from "./csvPlugin.js"; diff --git a/app/packages/language-csv/tsconfig.json b/app/packages/language-csv/tsconfig.json new file mode 100644 index 00000000..b11a3e26 --- /dev/null +++ b/app/packages/language-csv/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-common" }, + { "path": "../language-shared" } + ] +} diff --git a/app/packages/service-csv/package.json b/app/packages/service-csv/package.json new file mode 100644 index 00000000..6e2e9a87 --- /dev/null +++ b/app/packages/service-csv/package.json @@ -0,0 +1,26 @@ +{ + "name": "@mdeo/service-csv", + "version": "0.1.0", + "private": true, + "type": "module", + "files": ["dist", "src"], + "scripts": { + "build:static": "vite build", + "watch:static": "vite build --watch", + "start": "node dist/index.js", + "watch": "tsx watch src/index.ts" + }, + "dependencies": { + "@mdeo/language-csv": "^0.1.0", + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "tsx": "^4.21.0", + "vite": "^7.3.1" + } +} diff --git a/app/packages/service-csv/src/index.ts b/app/packages/service-csv/src/index.ts new file mode 100644 index 00000000..8d97c5f3 --- /dev/null +++ b/app/packages/service-csv/src/index.ts @@ -0,0 +1,67 @@ +import { Table } from "lucide"; +import { + startLanguageService, + parseServiceConfigFromEnv, + initializePluginContext, + AST_HANDLER_KEY, + astHandler, + type ServicePluginDefinition, + type LanguageServiceConfig +} from "@mdeo/service-common"; +import { + convertIcon, + defaultLanguageConfiguration, + defaultMonarchTokenProvider, + serializeMonarchTokensProvider, + type ExternalReferenceAdditionalServices +} from "@mdeo/language-common"; +import type { LanguagePlugin } from "@mdeo/plugin"; + +const csvLanguagePlugin: LanguagePlugin = { + id: "csv", + name: "CSV", + extension: ".csv", + newFileAction: true, + icon: convertIcon(Table), + serverPlugin: { + import: "language.js" + }, + graphicalEditorPlugin: undefined, + textualEditorPlugin: { + languageConfiguration: defaultLanguageConfiguration, + monarchTokensProvider: serializeMonarchTokensProvider({ + ...defaultMonarchTokenProvider, + keywords: [] + }) + }, + isGenerated: false +}; + +const csvServicePlugin: ServicePluginDefinition = { + id: "csv-service", + name: "CSV", + description: "Language support for CSV data files (.csv files)", + icon: convertIcon(Table), + languagePlugins: [csvLanguagePlugin], + contributionPlugins: [] +}; + +initializePluginContext(); + +const { csvPluginProvider } = await import("@mdeo/language-csv"); + +const envConfig = parseServiceConfigFromEnv(); + +const csvLanguageConfig: LanguageServiceConfig = { + languagePlugin: csvLanguagePlugin, + languagePluginProvider: csvPluginProvider, + fileDataHandlers: { + [AST_HANDLER_KEY]: astHandler + } +}; + +await startLanguageService({ + ...envConfig, + plugin: csvServicePlugin, + languages: [csvLanguageConfig] +}); diff --git a/app/packages/service-csv/src/served/language.ts b/app/packages/service-csv/src/served/language.ts new file mode 100644 index 00000000..89853725 --- /dev/null +++ b/app/packages/service-csv/src/served/language.ts @@ -0,0 +1,3 @@ +import { csvPluginProvider } from "@mdeo/language-csv"; + +export default csvPluginProvider; diff --git a/app/packages/service-csv/tsconfig.json b/app/packages/service-csv/tsconfig.json new file mode 100644 index 00000000..b3bc6233 --- /dev/null +++ b/app/packages/service-csv/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-csv" }, + { "path": "../language-common" }, + { "path": "../language-shared" }, + { "path": "../plugin" }, + { "path": "../service-common" } + ] +} diff --git a/app/packages/service-csv/vite.config.ts b/app/packages/service-csv/vite.config.ts new file mode 100644 index 00000000..1df064ba --- /dev/null +++ b/app/packages/service-csv/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vite"; +import { resolve } from "path"; + +export default defineConfig({ + build: { + lib: { + entry: { + language: resolve(__dirname, "src/served/language.ts") + }, + formats: ["es"] + }, + outDir: "static", + emptyOutDir: true, + sourcemap: true + } +}); diff --git a/app/tsconfig.build.json b/app/tsconfig.build.json index 42fc189f..9e024644 100644 --- a/app/tsconfig.build.json +++ b/app/tsconfig.build.json @@ -5,6 +5,12 @@ { "path": "packages/plugin" }, + { + "path": "packages/language-csv" + }, + { + "path": "packages/service-csv" + }, { "path": "packages/language-common" }, diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index 84c897b3..b34221d1 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -108,6 +108,7 @@ services: PLUGIN_CONFIG_OPTIMIZATION_SERVICE: http://service-config-optimization:3000 PLUGIN_CONFIG_MDEO_SERVICE: http://service-config-mdeo:3000 BACKEND_SERVICE: http://backend:8080 + PLUGIN_CSV_SERVICE: http://service-csv:3000 ports: - "4242:80" depends_on: @@ -148,6 +149,15 @@ services: start_period: 30s restart: unless-stopped + service-csv: + build: + context: . + dockerfile: infra/docker/service-csv/Dockerfile + environment: + PLUGIN_BACKEND_URL: "http://backend:8080" + networks: + - mdeo + service-model: build: context: .. diff --git a/infra/docker/service-csv/Dockerfile b/infra/docker/service-csv/Dockerfile new file mode 100644 index 00000000..1398db57 --- /dev/null +++ b/infra/docker/service-csv/Dockerfile @@ -0,0 +1,25 @@ +FROM node:22-alpine AS service-csv-builder + +WORKDIR /build + +COPY app/ ./ + +RUN npm ci +RUN npm run build:packages +RUN npm run -w @mdeo/service-csv build:static + +FROM node:22-alpine AS service-csv + +WORKDIR /app + +COPY --from=service-csv-builder /build/packages/service-csv/package.json ./package.json +COPY --from=service-csv-builder /build/packages/service-csv/dist ./dist +COPY --from=service-csv-builder /build/packages/service-csv/static ./static +COPY --from=service-csv-builder /build/node_modules ./node_modules + +ENV PORT=3000 +ENV HOST=0.0.0.0 + +EXPOSE 3000 + +CMD ["node", "dist/index.js"] diff --git a/infra/docker/workbench/nginx.conf b/infra/docker/workbench/nginx.conf index d50b9991..ee7f7b01 100644 --- a/infra/docker/workbench/nginx.conf +++ b/infra/docker/workbench/nginx.conf @@ -52,6 +52,12 @@ server { add_header Cross-Origin-Embedder-Policy "require-corp" always; } + location ^~ /plugin/csv/ { + proxy_pass ${PLUGIN_CSV_SERVICE}/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + # Model-transformation plugin location ^~ /plugin/model-transformation/ { proxy_pass ${PLUGIN_MODEL_TRANSFORMATION_SERVICE}/; From 7275d68e9ed2b4d4bfd36e07daf4da40a2cd6cc0 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:12:28 -0400 Subject: [PATCH 10/41] Fix service-csv Dockerfile and compose config to match existing service pattern --- infra/docker-compose-dev.yaml | 18 ++++++++++++++---- infra/docker/service-csv/Dockerfile | 19 +++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index b34221d1..2fb5fa2d 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -151,12 +151,22 @@ services: service-csv: build: - context: . + context: .. dockerfile: infra/docker/service-csv/Dockerfile + ports: + - "3007:3000" environment: - PLUGIN_BACKEND_URL: "http://backend:8080" - networks: - - mdeo + PORT: "3000" + HOST: "0.0.0.0" + BACKEND_API_URL: "http://backend:8080/api" + MAX_LANGIUM_INSTANCES: "5" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped service-model: build: diff --git a/infra/docker/service-csv/Dockerfile b/infra/docker/service-csv/Dockerfile index 1398db57..897bf478 100644 --- a/infra/docker/service-csv/Dockerfile +++ b/infra/docker/service-csv/Dockerfile @@ -1,22 +1,25 @@ -FROM node:22-alpine AS service-csv-builder +FROM node:24-alpine AS builder WORKDIR /build - COPY app/ ./ RUN npm ci RUN npm run build:packages RUN npm run -w @mdeo/service-csv build:static -FROM node:22-alpine AS service-csv +FROM node:24-alpine + +RUN apk add --no-cache curl + +WORKDIR /workspace -WORKDIR /app +COPY --from=builder /build/packages /workspace/packages +COPY --from=builder /build/node_modules /workspace/node_modules +COPY --from=builder /build/package.json /workspace/ -COPY --from=service-csv-builder /build/packages/service-csv/package.json ./package.json -COPY --from=service-csv-builder /build/packages/service-csv/dist ./dist -COPY --from=service-csv-builder /build/packages/service-csv/static ./static -COPY --from=service-csv-builder /build/node_modules ./node_modules +WORKDIR /workspace/packages/service-csv +ENV NODE_ENV=production ENV PORT=3000 ENV HOST=0.0.0.0 From bfced6b483e2f943d0dd6b7ae74d1f3e7d5c5dc4 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:37:08 -0400 Subject: [PATCH 11/41] Remove hardcoded port mapping for service-csv to avoid conflicts --- infra/docker-compose-dev.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index 2fb5fa2d..9d033ecb 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -153,8 +153,6 @@ services: build: context: .. dockerfile: infra/docker/service-csv/Dockerfile - ports: - - "3007:3000" environment: PORT: "3000" HOST: "0.0.0.0" @@ -330,7 +328,7 @@ services: args: SERVICE_VERSION: ${SERVICE_VERSION:-} ports: - - "3007:3000" + - "3008:3000" environment: PORT: "3000" HOST: "0.0.0.0" From 77b5e20003c160fb8cad3cd3aa2d278a37dd904d Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:41:00 -0400 Subject: [PATCH 12/41] Final CSV import: updated routes, inference engine, and tests --- app/package-lock.json | 43 +++- .../mdeo/backend/routes/CsvImportRoutes.kt | 13 +- .../mdeo/metamodel/csv/CsvModelInference.kt | 171 ++++++-------- .../csv/CsvModelInferenceManualRun.kt | 17 +- .../metamodel/csv/CsvModelInferenceTest.kt | 209 +++++++++--------- 5 files changed, 234 insertions(+), 219 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index f4c3b834..e1cf873e 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1636,6 +1636,10 @@ "resolved": "packages/language-config-optimization", "link": true }, + "node_modules/@mdeo/language-csv": { + "resolved": "packages/language-csv", + "link": true + }, "node_modules/@mdeo/language-expression": { "resolved": "packages/language-expression", "link": true @@ -1700,6 +1704,10 @@ "resolved": "packages/service-config-optimization", "link": true }, + "node_modules/@mdeo/service-csv": { + "resolved": "packages/service-csv", + "link": true + }, "node_modules/@mdeo/service-metamodel": { "resolved": "packages/service-metamodel", "link": true @@ -7784,7 +7792,8 @@ "version": "0.1.0", "dependencies": { "@mdeo/plugin": "^0.1.0", - "lucide": "^0.575.0" + "lucide": "^0.575.0", + "munkres": "^2.0.4" }, "devDependencies": { "@eclipse-glsp/graph": "^2.6.0", @@ -7851,6 +7860,18 @@ "vscode-languageserver-types": "^3.17.5" } }, + "packages/language-csv": { + "name": "@mdeo/language-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, "packages/language-expression": { "name": "@mdeo/language-expression", "version": "0.1.0", @@ -8098,6 +8119,23 @@ "vite": "~7.3.1" } }, + "packages/service-csv": { + "name": "@mdeo/service-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "tsx": "^4.21.0", + "vite": "^7.3.1" + } + }, "packages/service-metamodel": { "name": "@mdeo/service-metamodel", "version": "0.1.0", @@ -8108,6 +8146,7 @@ "@mdeo/language-metamodel": "^0.1.0", "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", + "@mdeo/protocol-metamodel": "^0.1.0", "@mdeo/service-common": "^0.1.0", "lucide": "^0.575.0" }, @@ -8125,6 +8164,7 @@ "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", "@mdeo/plugin": "^0.1.0", + "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", "lucide": "^0.575.0" }, @@ -8146,6 +8186,7 @@ "@mdeo/language-model-transformation": "^0.1.0", "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", + "@mdeo/protocol-model-transformation": "^0.1.0", "@mdeo/service-common": "^0.1.0" }, "devDependencies": { diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt index b03407c6..06a82de9 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt @@ -46,8 +46,17 @@ fun Route.csvImportRoutes( return@post } + val metamodelPath = call.request.queryParameters["metamodelPath"] + if (metamodelPath.isNullOrBlank()) { + call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Missing metamodelPath parameter")) + return@post + } + val className = call.request.queryParameters["className"] - ?: basePath.substringAfterLast('/').ifBlank { "ImportedCsvClass" } + if (className.isNullOrBlank()) { + call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Missing className parameter")) + return@post + } val csvText = call.receiveText() if (csvText.isBlank()) { @@ -55,7 +64,7 @@ fun Route.csvImportRoutes( return@post } - val result = csvImportService.importCsv(projectId, csvText, basePath, className) + val result = csvImportService.importCsv(projectId, csvText, basePath, className, metamodelPath) call.respondApiResult(result) } } diff --git a/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt b/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt index 1e27f8c7..5a2fd1d2 100644 --- a/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt +++ b/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt @@ -1,43 +1,42 @@ package com.mdeo.metamodel.csv import com.mdeo.metamodel.data.ClassData -import com.mdeo.metamodel.data.EnumData import com.mdeo.metamodel.data.MetamodelData import com.mdeo.metamodel.data.ModelData import com.mdeo.metamodel.data.ModelDataInstance import com.mdeo.metamodel.data.ModelDataPropertyValue -import com.mdeo.metamodel.data.MultiplicityData -import com.mdeo.metamodel.data.PropertyData object CsvModelInference { - const val DEFAULT_ENUM_MAX_DISTINCT = 20 - - data class InferenceResult( - val metamodel: MetamodelData, + data class ImportResult( val model: ModelData, val warnings: List ) - fun inferFromCsv( + fun importFromCsv( csvText: String, className: String, - metamodelPath: String, - enumMaxDistinct: Int = DEFAULT_ENUM_MAX_DISTINCT - ): InferenceResult { + metamodel: MetamodelData, + metamodelPath: String + ): ImportResult { + val targetClass = metamodel.classes.find { it.name == className } + ?: throw IllegalArgumentException( + "Class '$className' not found in metamodel. Available classes: ${metamodel.classes.map { it.name }.joinToString()}" + ) + val rows = parseCsv(csvText) require(rows.isNotEmpty()) { "CSV has no header row" } val header = rows.first() require(header.isNotEmpty()) { "CSV header row is empty" } - val duplicates = header.groupingBy { it }.eachCount().filterValues { it > 1 }.keys - require(duplicates.isEmpty()) { "CSV header has duplicate column names: ${duplicates.joinToString()}" } val dataRows = rows.drop(1) require(dataRows.isNotEmpty()) { "CSV has a header row but no data rows" } val warnings = mutableListOf() + validateHeaders(header, targetClass, warnings) + val normalizedRows = dataRows.mapIndexed { rowIndex, row -> when { row.size == header.size -> row @@ -52,59 +51,16 @@ object CsvModelInference { } } - val columns: List> = header.indices.map { colIndex -> - normalizedRows.map { it[colIndex] } - } - - val properties = mutableListOf() - val enums = mutableListOf() - - header.forEachIndexed { colIndex, rawColumnName -> - val propertyName = sanitizePropertyName(rawColumnName, fallback = "column${colIndex + 1}") - val values = columns[colIndex] - val nonBlankValues = values.filter { it.isNotBlank() } - val isOptional = nonBlankValues.size < values.size - - val inferred = inferColumnType(propertyName, nonBlankValues, enumMaxDistinct) - if (inferred is ColumnType.Enum) { - enums.add(EnumData(name = inferred.enumName, entries = inferred.entries)) - } - - properties.add( - PropertyData( - name = propertyName, - enumType = (inferred as? ColumnType.Enum)?.enumName, - primitiveType = (inferred as? ColumnType.Primitive)?.typeName, - multiplicity = if (isOptional) MultiplicityData.optional() else MultiplicityData.single() - ) - ) - } - - val sanitizedClassName = sanitizePropertyName(className, fallback = "ImportedCsvClass") - .let { it.replaceFirstChar(Char::uppercaseChar) } - - val metamodel = MetamodelData( - path = metamodelPath, - classes = listOf( - ClassData( - name = sanitizedClassName, - isAbstract = false, - properties = properties - ) - ), - enums = enums - ) - val instances = normalizedRows.mapIndexed { rowIndex, row -> val instanceProperties = header.indices.associate { colIndex -> - val propertyName = properties[colIndex].name - val property = properties[colIndex] + val columnName = header[colIndex].trim() + val property = targetClass.properties.find { it.name == columnName } val rawValue = row[colIndex] - propertyName to cellToPropertyValue(rawValue, property) + columnName to cellToPropertyValue(rawValue, property, metamodel) } ModelDataInstance( - name = "${sanitizedClassName}_$rowIndex", - className = sanitizedClassName, + name = "${className}_$rowIndex", + className = className, properties = instanceProperties ) } @@ -115,67 +71,70 @@ object CsvModelInference { links = emptyList() ) - return InferenceResult(metamodel = metamodel, model = model, warnings = warnings) + return ImportResult(model = model, warnings = warnings) } - private sealed class ColumnType { - data class Primitive(val typeName: String) : ColumnType() - data class Enum(val enumName: String, val entries: List) : ColumnType() - } - - private fun inferColumnType(propertyName: String, nonBlankValues: List, enumMaxDistinct: Int): ColumnType { - if (nonBlankValues.isEmpty()) { - return ColumnType.Primitive("string") + private fun validateHeaders( + header: List, + targetClass: ClassData, + warnings: MutableList + ) { + val propertyNames = targetClass.properties.map { it.name }.toSet() + + header.forEach { columnName -> + val trimmed = columnName.trim() + if (trimmed !in propertyNames) { + warnings.add("Column '$trimmed' does not match any property in class '${targetClass.name}' and will be stored as-is.") + } } - if (nonBlankValues.all { it.isIntegerLiteral() }) { - return ColumnType.Primitive("int") - } - if (nonBlankValues.all { it.isNumberLiteral() }) { - return ColumnType.Primitive("double") + val missingRequired = targetClass.properties.filter { property -> + property.multiplicity.isRequired() && header.none { it.trim() == property.name } } - if (nonBlankValues.all { it.isBooleanLiteral() }) { - return ColumnType.Primitive("boolean") - } - - val distinctValues = nonBlankValues.distinct() - val isRepeated = distinctValues.size < nonBlankValues.size - if (distinctValues.size <= enumMaxDistinct && isRepeated) { - val enumName = propertyName.replaceFirstChar(Char::uppercaseChar) + "Enum" - return ColumnType.Enum(enumName, distinctValues.map { sanitizeEnumEntry(it) }) + if (missingRequired.isNotEmpty()) { + throw IllegalArgumentException( + "CSV is missing required properties for class '${targetClass.name}': ${missingRequired.map { it.name }.joinToString()}" + ) } - - return ColumnType.Primitive("string") } - private fun String.isIntegerLiteral(): Boolean = toLongOrNull() != null - - private fun String.isNumberLiteral(): Boolean = toDoubleOrNull() != null - - private fun String.isBooleanLiteral(): Boolean = - equals("true", ignoreCase = true) || equals("false", ignoreCase = true) - - private fun cellToPropertyValue(rawValue: String, property: PropertyData): ModelDataPropertyValue { + private fun cellToPropertyValue( + rawValue: String, + property: com.mdeo.metamodel.data.PropertyData?, + metamodel: MetamodelData + ): ModelDataPropertyValue { if (rawValue.isBlank()) { return ModelDataPropertyValue.NullValue } + + if (property == null) { + return ModelDataPropertyValue.StringValue(rawValue) + } + return when { - property.enumType != null -> ModelDataPropertyValue.EnumValue(sanitizeEnumEntry(rawValue)) - property.primitiveType == "int" -> ModelDataPropertyValue.NumberValue(rawValue.toLong().toDouble()) - property.primitiveType == "double" -> ModelDataPropertyValue.NumberValue(rawValue.toDouble()) - property.primitiveType == "boolean" -> ModelDataPropertyValue.BooleanValue(rawValue.equals("true", ignoreCase = true)) + property.enumType != null -> { + val enumDef = metamodel.enums.find { it.name == property.enumType } + if (enumDef != null && rawValue !in enumDef.entries) { + ModelDataPropertyValue.StringValue(rawValue) + } else { + ModelDataPropertyValue.EnumValue(rawValue) + } + } + property.primitiveType == "int" || property.primitiveType == "long" -> { + rawValue.toLongOrNull()?.let { ModelDataPropertyValue.NumberValue(it.toDouble()) } + ?: ModelDataPropertyValue.StringValue(rawValue) + } + property.primitiveType == "double" || property.primitiveType == "float" -> { + rawValue.toDoubleOrNull()?.let { ModelDataPropertyValue.NumberValue(it) } + ?: ModelDataPropertyValue.StringValue(rawValue) + } + property.primitiveType == "boolean" -> { + ModelDataPropertyValue.BooleanValue(rawValue.equals("true", ignoreCase = true)) + } else -> ModelDataPropertyValue.StringValue(rawValue) } } - private fun sanitizePropertyName(raw: String, fallback: String): String { - val cleaned = raw.trim().map { c -> if (c.isLetterOrDigit() || c == '_') c else '_' }.joinToString("") - val withoutLeadingDigit = if (cleaned.firstOrNull()?.isDigit() == true) "_$cleaned" else cleaned - return withoutLeadingDigit.ifBlank { fallback } - } - - private fun sanitizeEnumEntry(raw: String): String = sanitizePropertyName(raw, fallback = "UNKNOWN") - private fun parseCsv(text: String): List> { val rows = mutableListOf>() var currentRow = mutableListOf() diff --git a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt index d120cc35..28470de5 100644 --- a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt +++ b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt @@ -1,30 +1,35 @@ package com.mdeo.metamodel.csv +import com.mdeo.metamodel.data.MetamodelData import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import java.nio.file.Path class CsvModelInferenceManualRun { @Test - fun `print inferred metamodel and model for a real csv file`() { + @Disabled("Manual run only (requires local CSV and metamodel files).") + fun `import csv onto existing metamodel`() { val csvPath = Path.of("/Users/amirguliyev/Downloads/employees.csv") + val metamodelPath = Path.of("/Users/amirguliyev/Downloads/employees.metamodel") + val csvText = csvPath.toFile().readText() + val metamodel = Json.decodeFromString(metamodelPath.toFile().readText()) val className = csvPath.fileName.toString().removeSuffix(".csv") + .replaceFirstChar { it.uppercaseChar() } - val result = CsvModelInference.inferFromCsv( + val result = CsvModelInference.importFromCsv( csvText = csvText, className = className, - metamodelPath = "/$className.metamodel" + metamodel = metamodel, + metamodelPath = metamodelPath.toString() ) val json = Json { prettyPrint = true } - println("===== METAMODEL =====") - println(json.encodeToString(result.metamodel)) - println("===== MODEL =====") println(json.encodeToString(result.model)) diff --git a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt index c655f6e1..d4de7ddd 100644 --- a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt +++ b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt @@ -1,157 +1,158 @@ package com.mdeo.metamodel.csv +import com.mdeo.metamodel.data.ClassData +import com.mdeo.metamodel.data.EnumData +import com.mdeo.metamodel.data.MetamodelData import com.mdeo.metamodel.data.ModelDataPropertyValue +import com.mdeo.metamodel.data.MultiplicityData +import com.mdeo.metamodel.data.PropertyData import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test class CsvModelInferenceTest { - @Test - fun `infers int, double, boolean and string columns`() { - val csv = """ - id,price,active,name - 1,9.99,true,Widget - 2,12.5,false,Gadget - 3,7,true,Gizmo - """.trimIndent() + private fun metamodel(vararg classes: ClassData, enums: List = emptyList()) = + MetamodelData(path = "/test.metamodel", classes = classes.toList(), enums = enums) - val result = CsvModelInference.inferFromCsv(csv, className = "product", metamodelPath = "/products.metamodel") + private fun cls(name: String, vararg properties: PropertyData) = + ClassData(name = name, isAbstract = false, properties = properties.toList()) - val productClass = result.metamodel.classes.single() - assertEquals("Product", productClass.name) - - val byName = productClass.properties.associateBy { it.name } - assertEquals("int", byName["id"]?.primitiveType) - assertEquals("double", byName["price"]?.primitiveType) - assertEquals("boolean", byName["active"]?.primitiveType) - assertEquals("string", byName["name"]?.primitiveType) - - assertEquals(3, result.model.instances.size) - assertEquals("Product_0", result.model.instances[0].name) - assertEquals( - ModelDataPropertyValue.NumberValue(9.99), - result.model.instances[0].properties["price"] + private fun prop(name: String, primitiveType: String, required: Boolean = true) = + PropertyData( + name = name, + primitiveType = primitiveType, + enumType = null, + multiplicity = if (required) MultiplicityData.single() else MultiplicityData.optional() ) - assertEquals( - ModelDataPropertyValue.BooleanValue(true), - result.model.instances[0].properties["active"] + + private fun enumProp(name: String, enumType: String, required: Boolean = true) = + PropertyData( + name = name, + primitiveType = null, + enumType = enumType, + multiplicity = if (required) MultiplicityData.single() else MultiplicityData.optional() ) - assertEquals("/products.metamodel", result.model.metamodelPath) - } @Test - fun `infers enum for low-cardinality repeated text column`() { + fun `maps columns to metamodel properties by name`() { + val mm = metamodel(cls("Employee", + prop("id", "int"), + prop("name", "string"), + prop("salary", "double"), + prop("active", "boolean") + )) + val csv = """ - id,status - 1,OPEN - 2,CLOSED - 3,OPEN - 4,OPEN - 5,CLOSED + id,name,salary,active + 1,Alice,95000.0,true + 2,Bob,72000.5,false """.trimIndent() - val result = CsvModelInference.inferFromCsv(csv, className = "ticket", metamodelPath = "/t.metamodel") - - val statusProp = result.metamodel.classes.single().properties.first { it.name == "status" } - assertNotNull(statusProp.enumType) - assertNull(statusProp.primitiveType) + val result = CsvModelInference.importFromCsv(csv, "Employee", mm, "/test.metamodel") - val enumDef = result.metamodel.enums.single() - assertEquals(setOf("OPEN", "CLOSED"), enumDef.entries.toSet()) - - val firstStatus = result.model.instances[0].properties["status"] - assertTrue(firstStatus is ModelDataPropertyValue.EnumValue) - assertEquals("OPEN", (firstStatus as ModelDataPropertyValue.EnumValue).enumEntry) + assertEquals(2, result.model.instances.size) + val alice = result.model.instances[0] + assertEquals("Employee_0", alice.name) + assertEquals(ModelDataPropertyValue.NumberValue(1.0), alice.properties["id"]) + assertEquals(ModelDataPropertyValue.StringValue("Alice"), alice.properties["name"]) + assertEquals(ModelDataPropertyValue.NumberValue(95000.0), alice.properties["salary"]) + assertEquals(ModelDataPropertyValue.BooleanValue(true), alice.properties["active"]) } @Test - fun `does not infer enum for high-cardinality or non-repeated text`() { - val csv = """ - id,name - 1,Alice - 2,Bob - 3,Carol - """.trimIndent() + fun `maps enum values from metamodel`() { + val mm = metamodel( + cls("Ticket", enumProp("status", "StatusEnum")), + enums = listOf(EnumData(name = "StatusEnum", entries = listOf("OPEN", "CLOSED"))) + ) + + val csv = "status\nOPEN\nCLOSED\nOPEN" + val result = CsvModelInference.importFromCsv(csv, "Ticket", mm, "/test.metamodel") - val result = CsvModelInference.inferFromCsv(csv, className = "person", metamodelPath = "/p.metamodel") - val nameProp = result.metamodel.classes.single().properties.first { it.name == "name" } - assertEquals("string", nameProp.primitiveType) - assertNull(nameProp.enumType) + assertEquals(ModelDataPropertyValue.EnumValue("OPEN"), result.model.instances[0].properties["status"]) + assertEquals(ModelDataPropertyValue.EnumValue("CLOSED"), result.model.instances[1].properties["status"]) } @Test - fun `marks columns with blank values as optional`() { - val csv = """ - id,nickname - 1,Ace - 2, - 3,Trey - """.trimIndent() - - val result = CsvModelInference.inferFromCsv(csv, className = "player", metamodelPath = "/pl.metamodel") - val props = result.metamodel.classes.single().properties.associateBy { it.name } + fun `treats blank cells as null`() { + val mm = metamodel(cls("Person", + prop("name", "string"), + prop("nickname", "string", required = false) + )) - assertTrue(props["id"]!!.multiplicity.isRequired()) - assertFalse(props["nickname"]!!.multiplicity.isRequired()) + val csv = "name,nickname\nAlice,Ace\nBob," + val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") assertEquals(ModelDataPropertyValue.NullValue, result.model.instances[1].properties["nickname"]) } @Test - fun `handles quoted fields with embedded commas and escaped quotes`() { - val csv = "id,description\n1,\"Contains, a comma\"\n2,\"Has \"\"quotes\"\" inside\"" - - val result = CsvModelInference.inferFromCsv(csv, className = "item", metamodelPath = "/i.metamodel") - val descriptions = result.model.instances.map { - (it.properties["description"] as ModelDataPropertyValue.StringValue).value - } + fun `warns about unknown columns`() { + val mm = metamodel(cls("Person", prop("name", "string"))) + val csv = "name,unknown_col\nAlice,extra" + val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") - assertEquals(listOf("Contains, a comma", "Has \"quotes\" inside"), descriptions) + assertTrue(result.warnings.any { it.contains("unknown_col") }) } @Test - fun `sanitizes header names into valid identifiers`() { - val csv = """ - Item ID,Item Name,Price (USD) - 1,Widget,9.99 - """.trimIndent() - - val result = CsvModelInference.inferFromCsv(csv, className = "catalog item", metamodelPath = "/c.metamodel") - val propNames = result.metamodel.classes.single().properties.map { it.name } + fun `throws when class not found in metamodel`() { + val mm = metamodel(cls("Person", prop("name", "string"))) + val csv = "name\nAlice" - assertEquals(listOf("Item_ID", "Item_Name", "Price__USD_"), propNames) - assertEquals("Catalog_item", result.metamodel.classes.single().name) - } - - @Test - fun `rejects empty csv`() { assertThrows(IllegalArgumentException::class.java) { - CsvModelInference.inferFromCsv("", className = "x", metamodelPath = "/x.metamodel") + CsvModelInference.importFromCsv(csv, "NonExistent", mm, "/test.metamodel") } } @Test - fun `rejects header-only csv with no data rows`() { - assertThrows(IllegalArgumentException::class.java) { - CsvModelInference.inferFromCsv("a,b,c", className = "x", metamodelPath = "/x.metamodel") - } - } + fun `throws when required property is missing from csv`() { + val mm = metamodel(cls("Person", + prop("name", "string"), + prop("id", "int") + )) + val csv = "name\nAlice" - @Test - fun `rejects duplicate column names`() { - val csv = "a,a,b\n1,2,3" assertThrows(IllegalArgumentException::class.java) { - CsvModelInference.inferFromCsv(csv, className = "x", metamodelPath = "/x.metamodel") + CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") } } @Test fun `pads ragged short rows and warns`() { - val csv = "a,b,c\n1,2,3\n4,5" - val result = CsvModelInference.inferFromCsv(csv, className = "x", metamodelPath = "/x.metamodel") + val mm = metamodel(cls("Person", + prop("name", "string"), + prop("age", "int", required = false) + )) + val csv = "name,age\nAlice,30\nBob" + val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") assertEquals(2, result.model.instances.size) - assertEquals(ModelDataPropertyValue.NullValue, result.model.instances[1].properties["c"]) + assertEquals(ModelDataPropertyValue.NullValue, result.model.instances[1].properties["age"]) assertTrue(result.warnings.any { it.contains("fewer columns") }) } + + @Test + fun `handles quoted fields with embedded commas`() { + val mm = metamodel(cls("Item", prop("description", "string"))) + val csv = "description\n\"Contains, a comma\"" + val result = CsvModelInference.importFromCsv(csv, "Item", mm, "/test.metamodel") + + assertEquals( + ModelDataPropertyValue.StringValue("Contains, a comma"), + result.model.instances[0].properties["description"] + ) + } + + @Test + fun `gracefully falls back to string for unparseable int`() { + val mm = metamodel(cls("Item", prop("count", "int"))) + val csv = "count\nnot_a_number" + val result = CsvModelInference.importFromCsv(csv, "Item", mm, "/test.metamodel") + + assertEquals( + ModelDataPropertyValue.StringValue("not_a_number"), + result.model.instances[0].properties["count"] + ) + } } From 3ee7d4e2eba1146b7e1ceb76d3e231c708340bbd Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 01:41:09 -0400 Subject: [PATCH 13/41] Add .DS_Store to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e1bc6426..5aaaa484 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ copilot/ tools/ infra/.env +.DS_Store From 79647eaa1b4c8042603e933186ef6d2dc408ff77 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 11:11:03 -0400 Subject: [PATCH 14/41] Add ModelContributionPlugin system and language-model-csv contribution plugin with import CSV syntax --- app/packages/language-model-csv/package.json | 17 +++ .../src/grammar/csvImportRules.ts | 21 ++++ .../src/grammar/csvImportTypes.ts | 15 +++ app/packages/language-model-csv/src/index.ts | 3 + .../src/plugin/modelCsvContributionPlugin.ts | 15 +++ app/packages/language-model-csv/tsconfig.json | 14 +++ .../language-model/src/grammar/modelRules.ts | 72 ++++--------- app/packages/language-model/src/index.ts | 1 + .../src/modelContributionPlugin.ts | 16 +++ .../language-model/src/modelPlugin.ts | 29 ++--- app/packages/service-model/package.json | 5 +- app/packages/service-model/src/index.ts | 100 ++++-------------- app/packages/service-model/tsconfig.json | 14 ++- infra/docker-compose-dev.yaml | 2 +- 14 files changed, 179 insertions(+), 145 deletions(-) create mode 100644 app/packages/language-model-csv/package.json create mode 100644 app/packages/language-model-csv/src/grammar/csvImportRules.ts create mode 100644 app/packages/language-model-csv/src/grammar/csvImportTypes.ts create mode 100644 app/packages/language-model-csv/src/index.ts create mode 100644 app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts create mode 100644 app/packages/language-model-csv/tsconfig.json create mode 100644 app/packages/language-model/src/modelContributionPlugin.ts diff --git a/app/packages/language-model-csv/package.json b/app/packages/language-model-csv/package.json new file mode 100644 index 00000000..56c1fd12 --- /dev/null +++ b/app/packages/language-model-csv/package.json @@ -0,0 +1,17 @@ +{ + "name": "@mdeo/language-model-csv", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist", "src"], + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-metamodel": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts new file mode 100644 index 00000000..5300dc7b --- /dev/null +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -0,0 +1,21 @@ +import { createRule, many, STRING, NEWLINE, ID, or, ref } from "@mdeo/language-common"; +import { Class } from "@mdeo/language-metamodel"; +import { CsvClassImport, CsvImportBlock } from "./csvImportTypes.js"; + +export const CsvClassImportRule = createRule("CsvClassImportRule") + .returns(CsvClassImport) + .as(({ set }) => [ + set("className", ref(Class, ID)), + "from", + set("file", STRING) + ]); + +export const CsvImportBlockRule = createRule("CsvImportBlockRule") + .returns(CsvImportBlock) + .as(({ add }) => [ + "import", + "CSV", + "{", + many(or(add("imports", CsvClassImportRule), NEWLINE)), + "}" + ]); diff --git a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts new file mode 100644 index 00000000..cb4fac64 --- /dev/null +++ b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts @@ -0,0 +1,15 @@ +import { createInterface, Optional, Ref, type ASTType } from "@mdeo/language-common"; +import { Class } from "@mdeo/language-metamodel"; + +export const CsvClassImport = createInterface("CsvClassImport").attrs({ + className: Ref(() => Class), + file: String +}); + +export type CsvClassImportType = ASTType; + +export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ + imports: [CsvClassImport] +}); + +export type CsvImportBlockType = ASTType; diff --git a/app/packages/language-model-csv/src/index.ts b/app/packages/language-model-csv/src/index.ts new file mode 100644 index 00000000..cd29dbd9 --- /dev/null +++ b/app/packages/language-model-csv/src/index.ts @@ -0,0 +1,3 @@ +export * from "./grammar/csvImportTypes.js"; +export * from "./grammar/csvImportRules.js"; +export * from "./plugin/modelCsvContributionPlugin.js"; diff --git a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts new file mode 100644 index 00000000..edbebe84 --- /dev/null +++ b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts @@ -0,0 +1,15 @@ +import { type ModelContributionPlugin, MODEL_CONTRIBUTION_PLUGIN_TYPE } from "@mdeo/language-model"; +import { CsvClassImportRule, CsvImportBlockRule } from "../grammar/csvImportRules.js"; + +export const MODEL_CSV_CONTRIBUTION_PLUGIN_ID = "model-csv"; + +export function createModelCsvContributionPlugin(): ModelContributionPlugin { + return { + type: MODEL_CONTRIBUTION_PLUGIN_TYPE, + id: MODEL_CSV_CONTRIBUTION_PLUGIN_ID, + name: "CSV Import", + additionalTerminals: [], + additionalRules: [CsvClassImportRule, CsvImportBlockRule], + topLevelRuleNames: [CsvImportBlockRule.name] + }; +} diff --git a/app/packages/language-model-csv/tsconfig.json b/app/packages/language-model-csv/tsconfig.json new file mode 100644 index 00000000..224fb6ed --- /dev/null +++ b/app/packages/language-model-csv/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "include": ["src"], + "references": [ + { "path": "../language-common" }, + { "path": "../language-model" }, + { "path": "../language-metamodel" } + ] +} diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index a2a0c52b..3ba9a25b 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -2,6 +2,8 @@ import { createRule, or, many, + optional, + group, ref, ID, STRING, @@ -12,8 +14,7 @@ import { ML_COMMENT, SL_COMMENT, HIDDEN_NEWLINE, - optional, - group + type ParserRule } from "@mdeo/language-common"; import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import { @@ -27,9 +28,7 @@ import { Model, MetamodelFileImport, LinkEnd, - Link, - CsvClassImport, - CsvImportBlock + Link } from "./modelTypes.js"; export const BOOLEAN = createRule("BOOLEAN") @@ -92,51 +91,24 @@ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") .returns(MetamodelFileImport) .as(({ set }) => ["using", set("file", STRING)]); -/** - * CSV class import rule. - * Format: ClassName from "path/to/file.csv" - */ -export const CsvClassImportRule = createRule("CsvClassImportRule") - .returns(CsvClassImport) - .as(({ set }) => [ - set("class", ref(Class, ID)), - "from", - set("file", STRING) - ]); +export function createModelRule(contributedTopLevelRules: ParserRule[] = []): ParserRule { + return createRule("ModelRule") + .returns(Model) + .as(({ add, set }) => [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many( + or( + ...contributedTopLevelRules, + add("objects", ObjectInstanceRule), + add("links", LinkRule), + NEWLINE + ) + ) + ]); +} -/** - * CSV import block rule. - * Format: - * import CSV { - * ClassName from "file.csv" - * } - */ -export const CsvImportBlockRule = createRule("CsvImportBlockRule") - .returns(CsvImportBlock) - .as(({ add }) => [ - "import", - "CSV", - "{", - many(or(add("imports", CsvClassImportRule), NEWLINE)), - "}" - ]); - -/** - * Model root rule. - * A model file starts with a metamodel import, then either: - * - A CSV import block (no hand-authored objects allowed), or - * - Hand-authored object instances and links - */ -export const ModelRule = createRule("ModelRule") - .returns(Model) - .as(({ add, set }) => [ - many(NEWLINE), - set("import", MetamodelFileImportRule), - many(NEWLINE), - or( - set("csvImport", CsvImportBlockRule), - many(or(add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) - ) - ]); +export const ModelRule = createModelRule(); export const ModelTerminals = [WS, HIDDEN_NEWLINE, ML_COMMENT, SL_COMMENT]; diff --git a/app/packages/language-model/src/index.ts b/app/packages/language-model/src/index.ts index 5a92df5e..39250fd6 100644 --- a/app/packages/language-model/src/index.ts +++ b/app/packages/language-model/src/index.ts @@ -54,3 +54,4 @@ export * from "./validation/baseModelValidator.js"; export * from "./validation/modelValidator.js"; export * from "./generatedModelPlugin.js"; export * from "./modelPlugin.js"; +export * from "./modelContributionPlugin.js"; diff --git a/app/packages/language-model/src/modelContributionPlugin.ts b/app/packages/language-model/src/modelContributionPlugin.ts new file mode 100644 index 00000000..a2e94773 --- /dev/null +++ b/app/packages/language-model/src/modelContributionPlugin.ts @@ -0,0 +1,16 @@ +import type { TerminalRule, ParserRule } from "@mdeo/language-common"; + +export const MODEL_CONTRIBUTION_PLUGIN_TYPE = "model-contribution-plugin"; + +export interface ModelContributionPlugin { + readonly type: typeof MODEL_CONTRIBUTION_PLUGIN_TYPE; + readonly id: string; + readonly name: string; + readonly additionalTerminals: TerminalRule[]; + readonly additionalRules: ParserRule[]; + readonly topLevelRuleNames: string[]; +} + +export namespace ModelContributionPlugin { + export const TYPE = MODEL_CONTRIBUTION_PLUGIN_TYPE; +} diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index 67dec144..7e38b7a1 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -3,7 +3,7 @@ import { type LangiumLanguagePlugin, type LangiumLanguagePluginProvider } from "@mdeo/language-common"; -import { ModelRule, ModelTerminals } from "./grammar/modelRules.js"; +import { createModelRule, ModelTerminals } from "./grammar/modelRules.js"; import { addExternalReferenceCollectionPhase, type ActionHandlerRegistryAdditionalServices, @@ -23,17 +23,25 @@ import { registerModelSerializers } from "./features/modelSerializers.js"; import { ModelDiagramModule } from "./features/diagram-server/modelDiagramModule.js"; import { registerModelValidationChecks } from "./validation/modelValidator.js"; import { ModelCompletionProvider } from "./features/modelCompletionProvider.js"; +import type { ModelContributionPlugin } from "./modelContributionPlugin.js"; +import type { ServerContributionPlugin } from "@mdeo/plugin"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; -/** - * The plugin for the Model language. - * Provides minimal language server functionality for .m files. - */ -function createModelPlugin(languageJsUrl?: string): LangiumLanguagePlugin { +function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is ModelContributionPlugin { + return plugin.id !== undefined && (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; +} + +function createModelPlugin(contributionPlugins: ServerContributionPlugin[], languageJsUrl?: string): LangiumLanguagePlugin { + const modelContributions = contributionPlugins.filter(isModelContributionPlugin); + const contributedTopLevelRules = modelContributions.flatMap(p => p.additionalRules.filter(r => p.topLevelRuleNames.includes(r.name))); + const contributedTerminals = modelContributions.flatMap(p => p.additionalTerminals); + + const ModelRule = createModelRule(contributedTopLevelRules); + return { rootRule: ModelRule, - additionalTerminals: ModelTerminals, + additionalTerminals: [...ModelTerminals, ...contributedTerminals], module: { parser: { TokenBuilder: () => new NewlineAwareTokenBuilder(new Set(["{"]), new Set(["("]), new Set(["}", ")"])), @@ -73,11 +81,8 @@ function createModelPlugin(languageJsUrl?: string): LangiumLanguagePlugin = { - create(_contributionPlugins, languageJsUrl): LangiumLanguagePlugin { - return createModelPlugin(languageJsUrl); + create(contributionPlugins, languageJsUrl): LangiumLanguagePlugin { + return createModelPlugin(contributionPlugins, languageJsUrl); } }; diff --git a/app/packages/service-model/package.json b/app/packages/service-model/package.json index a1e15b88..6b0a261e 100644 --- a/app/packages/service-model/package.json +++ b/app/packages/service-model/package.json @@ -20,11 +20,12 @@ "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", - "lucide": "^0.575.0" + "lucide": "^0.575.0", + "@mdeo/language-model-csv": "^0.1.0" }, "devDependencies": { "@types/node": "^25.3.1", "tsx": "^4.21.0", "vite": "^7.3.1" } -} +} \ No newline at end of file diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index aee5db1d..1d1b4610 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -1,47 +1,28 @@ -import { Network } from "lucide"; +import { BookOpen } from "lucide"; import { defaultLanguageConfiguration, defaultMonarchTokenProvider, serializeMonarchTokensProvider, - type ActionIconNode + convertIcon } from "@mdeo/language-common"; import { + startLanguageService, parseServiceConfigFromEnv, type ServiceConfig, type ServicePluginDefinition, type LanguageServiceConfig, initializePluginContext, astHandler, - AST_HANDLER_KEY, - startLanguageService + AST_HANDLER_KEY } from "@mdeo/service-common"; -import { convertIcon } from "@mdeo/language-common"; -import type { ModelServices, GeneratedModelServices } from "@mdeo/language-model"; +import type { ModelServices } from "@mdeo/language-model"; import type { LanguagePlugin } from "@mdeo/plugin"; -const icon: ActionIconNode = convertIcon(Network).map((entry) => { - if (entry[0] === "rect") { - return [ - "rect", - { - ...entry[1], - fill: "currentColor" - } - ]; - } else { - return entry; - } -}); - -/** - * Language plugin definition for the model language. - */ const modelLanguagePlugin: LanguagePlugin = { id: "model", name: "Model", extension: ".m", - newFileAction: true, - icon, + icon: convertIcon(BookOpen), serverPlugin: { import: "language.js" }, @@ -54,80 +35,45 @@ const modelLanguagePlugin: LanguagePlugin = { languageConfiguration: defaultLanguageConfiguration, monarchTokensProvider: serializeMonarchTokensProvider({ ...defaultMonarchTokenProvider, - keywords: ["using"] + keywords: ["using", "import", "CSV", "from"] }) }, isGenerated: false }; -/** - * Language plugin definition for the generated model language. - */ -const generatedModelLanguagePlugin: LanguagePlugin = { - id: "model_gen", - name: "Generated Model", - extension: ".m_gen", - newFileAction: false, - icon, - serverPlugin: { - import: "generatedLanguage.js" - }, - graphicalEditorPlugin: { - import: "editor.js", - stylesUrl: "styles.css", - stylesCls: "editor-model" - }, - textualEditorPlugin: undefined, - isGenerated: true -}; +initializePluginContext(); + +const { modelPluginProvider } = await import("@mdeo/language-model"); +const { createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); + +const envConfig = parseServiceConfigFromEnv(); -/** - * Plugin definition for the model service. - */ const modelServicePlugin: ServicePluginDefinition = { id: "model-service", name: "Model", - description: "Language support for model definitions (.m and .m_gen files)", - icon, - languagePlugins: [modelLanguagePlugin, generatedModelLanguagePlugin], + description: "Language support for model definitions (.m files)", + icon: convertIcon(BookOpen), + languagePlugins: [modelLanguagePlugin], contributionPlugins: [] }; -initializePluginContext(); - -const { modelPluginProvider } = await import("@mdeo/language-model"); -const { generatedModelPluginProvider } = await import("@mdeo/language-model"); -const { modelDataHandler, MODEL_DATA_HANDLER_KEY } = await import("./handler/modelDataHandler.js"); - -const envConfig = parseServiceConfigFromEnv(); +const csvContributionPlugin = createModelCsvContributionPlugin(); -/** - * Language configuration for the model language. - */ const modelLanguageConfig: LanguageServiceConfig = { languagePlugin: modelLanguagePlugin, - languagePluginProvider: modelPluginProvider, - fileDataHandlers: { - [AST_HANDLER_KEY]: astHandler, - [MODEL_DATA_HANDLER_KEY]: modelDataHandler - } -}; - -/** - * Language configuration for the generated model language. - */ -const generatedModelLanguageConfig: LanguageServiceConfig = { - languagePlugin: generatedModelLanguagePlugin, - languagePluginProvider: generatedModelPluginProvider, + languagePluginProvider: { + create: (serverContributionPlugins, languageJsUrl) => + modelPluginProvider.create([...serverContributionPlugins, csvContributionPlugin], languageJsUrl) + }, fileDataHandlers: { [AST_HANDLER_KEY]: astHandler } }; -const config: ServiceConfig = { +const config: ServiceConfig = { ...envConfig, plugin: modelServicePlugin, - languages: [modelLanguageConfig, generatedModelLanguageConfig] + languages: [modelLanguageConfig] }; await startLanguageService(config); diff --git a/app/packages/service-model/tsconfig.json b/app/packages/service-model/tsconfig.json index 94acbe66..015502c0 100644 --- a/app/packages/service-model/tsconfig.json +++ b/app/packages/service-model/tsconfig.json @@ -19,8 +19,16 @@ }, { "path": "../service-common" + }, + { + "path": "../language-model-csv" } ], - "include": ["src/**/*.ts"], - "exclude": ["dist", "node_modules"] -} + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "dist", + "node_modules" + ] +} \ No newline at end of file diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index 9d033ecb..fe4fc8be 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -87,7 +87,7 @@ services: ADMIN_PASSWORD: admin PLUGIN_BASE_URL: "" INTERNAL_PLUGIN_BASE_URL: "http://workbench:80" - DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo" + DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo,/plugin/csv" depends_on: postgres-backend: condition: service_healthy From 4afa5c93ff165a48ed7e421b1a002cdb61a3542d Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:06:53 -0400 Subject: [PATCH 15/41] Fix contribution plugin grammar merging and remove invalid TokenBuilder argument --- .../src/plugin/modelCsvContributionPlugin.ts | 3 +- .../language-model/src/grammar/modelRules.ts | 30 ++++++++++--------- .../src/modelContributionPlugin.ts | 1 + .../language-model/src/modelPlugin.ts | 12 ++++++-- 4 files changed, 28 insertions(+), 18 deletions(-) diff --git a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts index edbebe84..df6e15d4 100644 --- a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts +++ b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts @@ -10,6 +10,7 @@ export function createModelCsvContributionPlugin(): ModelContributionPlugin { name: "CSV Import", additionalTerminals: [], additionalRules: [CsvClassImportRule, CsvImportBlockRule], - topLevelRuleNames: [CsvImportBlockRule.name] + topLevelRuleNames: [CsvImportBlockRule.name], + keywords: ["import", "CSV", "from"] }; } diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index 3ba9a25b..82c8e44c 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -14,7 +14,8 @@ import { ML_COMMENT, SL_COMMENT, HIDDEN_NEWLINE, - type ParserRule + type ParserRule, + type RuleEntry } from "@mdeo/language-common"; import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import { @@ -94,19 +95,20 @@ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") export function createModelRule(contributedTopLevelRules: ParserRule[] = []): ParserRule { return createRule("ModelRule") .returns(Model) - .as(({ add, set }) => [ - many(NEWLINE), - set("import", MetamodelFileImportRule), - many(NEWLINE), - many( - or( - ...contributedTopLevelRules, - add("objects", ObjectInstanceRule), - add("links", LinkRule), - NEWLINE - ) - ) - ]); + .as(({ add, set }) => { + const topLevelAlternatives: RuleEntry[] = [ + ...contributedTopLevelRules, + add("objects", ObjectInstanceRule), + add("links", LinkRule), + NEWLINE + ]; + return [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or(...topLevelAlternatives)) + ]; + }); } export const ModelRule = createModelRule(); diff --git a/app/packages/language-model/src/modelContributionPlugin.ts b/app/packages/language-model/src/modelContributionPlugin.ts index a2e94773..9826a1d1 100644 --- a/app/packages/language-model/src/modelContributionPlugin.ts +++ b/app/packages/language-model/src/modelContributionPlugin.ts @@ -9,6 +9,7 @@ export interface ModelContributionPlugin { readonly additionalTerminals: TerminalRule[]; readonly additionalRules: ParserRule[]; readonly topLevelRuleNames: string[]; + readonly keywords: string[]; } export namespace ModelContributionPlugin { diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index 7e38b7a1..ccb64100 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -29,12 +29,14 @@ import type { ServerContributionPlugin } from "@mdeo/plugin"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is ModelContributionPlugin { - return plugin.id !== undefined && (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; + return (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; } function createModelPlugin(contributionPlugins: ServerContributionPlugin[], languageJsUrl?: string): LangiumLanguagePlugin { const modelContributions = contributionPlugins.filter(isModelContributionPlugin); - const contributedTopLevelRules = modelContributions.flatMap(p => p.additionalRules.filter(r => p.topLevelRuleNames.includes(r.name))); + const contributedTopLevelRules = modelContributions.flatMap(p => + p.additionalRules.filter(r => p.topLevelRuleNames.includes(r.name)) + ); const contributedTerminals = modelContributions.flatMap(p => p.additionalTerminals); const ModelRule = createModelRule(contributedTopLevelRules); @@ -44,7 +46,11 @@ function createModelPlugin(contributionPlugins: ServerContributionPlugin[], lang additionalTerminals: [...ModelTerminals, ...contributedTerminals], module: { parser: { - TokenBuilder: () => new NewlineAwareTokenBuilder(new Set(["{"]), new Set(["("]), new Set(["}", ")"])), + TokenBuilder: () => new NewlineAwareTokenBuilder( + new Set(["{"]), + new Set(["("]), + new Set(["}", ")"]) + ), ValueConverter: () => new IdValueConverter(), ParserConfig: () => ({ maxLookahead: 4 From e55bc44e8fd4e2cd31af5916e4f97eb839ad2a50 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:20:00 -0400 Subject: [PATCH 16/41] Register CSV contribution plugin as built-in to model language at startup --- app/packages/language-model/src/index.ts | 1 + app/packages/language-model/src/modelPlugin.ts | 16 +++++++++++++--- app/packages/service-model/src/index.ts | 11 ++++------- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/app/packages/language-model/src/index.ts b/app/packages/language-model/src/index.ts index 39250fd6..6025cae3 100644 --- a/app/packages/language-model/src/index.ts +++ b/app/packages/language-model/src/index.ts @@ -55,3 +55,4 @@ export * from "./validation/modelValidator.js"; export * from "./generatedModelPlugin.js"; export * from "./modelPlugin.js"; export * from "./modelContributionPlugin.js"; +export { registerModelContributionPlugin } from "./modelPlugin.js"; diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index ccb64100..4b9ca6f0 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -28,16 +28,26 @@ import type { ServerContributionPlugin } from "@mdeo/plugin"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; +const builtInContributionPlugins: ModelContributionPlugin[] = []; + +export function registerModelContributionPlugin(plugin: ModelContributionPlugin): void { + builtInContributionPlugins.push(plugin); +} + function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is ModelContributionPlugin { return (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; } function createModelPlugin(contributionPlugins: ServerContributionPlugin[], languageJsUrl?: string): LangiumLanguagePlugin { - const modelContributions = contributionPlugins.filter(isModelContributionPlugin); - const contributedTopLevelRules = modelContributions.flatMap(p => + const allContributions = [ + ...builtInContributionPlugins, + ...contributionPlugins.filter(isModelContributionPlugin) + ]; + + const contributedTopLevelRules = allContributions.flatMap(p => p.additionalRules.filter(r => p.topLevelRuleNames.includes(r.name)) ); - const contributedTerminals = modelContributions.flatMap(p => p.additionalTerminals); + const contributedTerminals = allContributions.flatMap(p => p.additionalTerminals); const ModelRule = createModelRule(contributedTopLevelRules); diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index 1d1b4610..99f448af 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -43,9 +43,11 @@ const modelLanguagePlugin: LanguagePlugin = { initializePluginContext(); -const { modelPluginProvider } = await import("@mdeo/language-model"); +const { modelPluginProvider, registerModelContributionPlugin } = await import("@mdeo/language-model"); const { createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); +registerModelContributionPlugin(createModelCsvContributionPlugin()); + const envConfig = parseServiceConfigFromEnv(); const modelServicePlugin: ServicePluginDefinition = { @@ -57,14 +59,9 @@ const modelServicePlugin: ServicePluginDefinition = { contributionPlugins: [] }; -const csvContributionPlugin = createModelCsvContributionPlugin(); - const modelLanguageConfig: LanguageServiceConfig = { languagePlugin: modelLanguagePlugin, - languagePluginProvider: { - create: (serverContributionPlugins, languageJsUrl) => - modelPluginProvider.create([...serverContributionPlugins, csvContributionPlugin], languageJsUrl) - }, + languagePluginProvider: modelPluginProvider, fileDataHandlers: { [AST_HANDLER_KEY]: astHandler } From 0afc194aadba68fc8315c7f6d9a0f9a7d1edc9b7 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:24:11 -0400 Subject: [PATCH 17/41] Hardcode CSV contribution directly in modelPlugin to avoid ESM module instance issues --- .../language-model/src/csv/csvImportRules.ts | 21 +++++++++++++++++++ .../language-model/src/csv/csvImportTypes.ts | 15 +++++++++++++ .../language-model/src/modelPlugin.ts | 19 +++++++++++------ 3 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 app/packages/language-model/src/csv/csvImportRules.ts create mode 100644 app/packages/language-model/src/csv/csvImportTypes.ts diff --git a/app/packages/language-model/src/csv/csvImportRules.ts b/app/packages/language-model/src/csv/csvImportRules.ts new file mode 100644 index 00000000..5300dc7b --- /dev/null +++ b/app/packages/language-model/src/csv/csvImportRules.ts @@ -0,0 +1,21 @@ +import { createRule, many, STRING, NEWLINE, ID, or, ref } from "@mdeo/language-common"; +import { Class } from "@mdeo/language-metamodel"; +import { CsvClassImport, CsvImportBlock } from "./csvImportTypes.js"; + +export const CsvClassImportRule = createRule("CsvClassImportRule") + .returns(CsvClassImport) + .as(({ set }) => [ + set("className", ref(Class, ID)), + "from", + set("file", STRING) + ]); + +export const CsvImportBlockRule = createRule("CsvImportBlockRule") + .returns(CsvImportBlock) + .as(({ add }) => [ + "import", + "CSV", + "{", + many(or(add("imports", CsvClassImportRule), NEWLINE)), + "}" + ]); diff --git a/app/packages/language-model/src/csv/csvImportTypes.ts b/app/packages/language-model/src/csv/csvImportTypes.ts new file mode 100644 index 00000000..cb4fac64 --- /dev/null +++ b/app/packages/language-model/src/csv/csvImportTypes.ts @@ -0,0 +1,15 @@ +import { createInterface, Optional, Ref, type ASTType } from "@mdeo/language-common"; +import { Class } from "@mdeo/language-metamodel"; + +export const CsvClassImport = createInterface("CsvClassImport").attrs({ + className: Ref(() => Class), + file: String +}); + +export type CsvClassImportType = ASTType; + +export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ + imports: [CsvClassImport] +}); + +export type CsvImportBlockType = ASTType; diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index 4b9ca6f0..d3974fee 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -25,14 +25,19 @@ import { registerModelValidationChecks } from "./validation/modelValidator.js"; import { ModelCompletionProvider } from "./features/modelCompletionProvider.js"; import type { ModelContributionPlugin } from "./modelContributionPlugin.js"; import type { ServerContributionPlugin } from "@mdeo/plugin"; +import { CsvClassImportRule, CsvImportBlockRule } from "./csv/csvImportRules.js"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; -const builtInContributionPlugins: ModelContributionPlugin[] = []; - -export function registerModelContributionPlugin(plugin: ModelContributionPlugin): void { - builtInContributionPlugins.push(plugin); -} +const CSV_CONTRIBUTION: ModelContributionPlugin = { + type: "model-contribution-plugin", + id: "model-csv", + name: "CSV Import", + additionalTerminals: [], + additionalRules: [CsvClassImportRule, CsvImportBlockRule], + topLevelRuleNames: ["CsvImportBlockRule"], + keywords: ["import", "CSV", "from"] +}; function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is ModelContributionPlugin { return (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; @@ -40,7 +45,7 @@ function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is function createModelPlugin(contributionPlugins: ServerContributionPlugin[], languageJsUrl?: string): LangiumLanguagePlugin { const allContributions = [ - ...builtInContributionPlugins, + CSV_CONTRIBUTION, ...contributionPlugins.filter(isModelContributionPlugin) ]; @@ -102,3 +107,5 @@ export const modelPluginProvider: LangiumLanguagePluginProvider = return createModelPlugin(contributionPlugins, languageJsUrl); } }; + +export function registerModelContributionPlugin(_plugin: ModelContributionPlugin): void {} From 9b836c2746a5858015edd6815fe09cfb47e894e6 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:27:11 -0400 Subject: [PATCH 18/41] Add import CSV syntax directly to model grammar as a first-class feature --- .../language-model/src/grammar/modelRules.ts | 60 ++++++++++++------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index 82c8e44c..ab39522f 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -14,8 +14,7 @@ import { ML_COMMENT, SL_COMMENT, HIDDEN_NEWLINE, - type ParserRule, - type RuleEntry + type ParserRule } from "@mdeo/language-common"; import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import { @@ -29,7 +28,9 @@ import { Model, MetamodelFileImport, LinkEnd, - Link + Link, + CsvClassImport, + CsvImportBlock } from "./modelTypes.js"; export const BOOLEAN = createRule("BOOLEAN") @@ -92,25 +93,40 @@ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") .returns(MetamodelFileImport) .as(({ set }) => ["using", set("file", STRING)]); -export function createModelRule(contributedTopLevelRules: ParserRule[] = []): ParserRule { - return createRule("ModelRule") - .returns(Model) - .as(({ add, set }) => { - const topLevelAlternatives: RuleEntry[] = [ - ...contributedTopLevelRules, - add("objects", ObjectInstanceRule), - add("links", LinkRule), - NEWLINE - ]; - return [ - many(NEWLINE), - set("import", MetamodelFileImportRule), - many(NEWLINE), - many(or(...topLevelAlternatives)) - ]; - }); -} +export const CsvClassImportRule = createRule("CsvClassImportRule") + .returns(CsvClassImport) + .as(({ set }) => [ + set("class", ref(Class, ID)), + "from", + set("file", STRING) + ]); + +export const CsvImportBlockRule = createRule("CsvImportBlockRule") + .returns(CsvImportBlock) + .as(({ add }) => [ + "import", + "CSV", + "{", + many(or(add("imports", CsvClassImportRule), NEWLINE)), + "}" + ]); -export const ModelRule = createModelRule(); +export const ModelRule = createRule("ModelRule") + .returns(Model) + .as(({ add, set }) => [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or( + set("csvImport", CsvImportBlockRule), + add("objects", ObjectInstanceRule), + add("links", LinkRule), + NEWLINE + )) + ]); + +export function createModelRule(_contributedTopLevelRules: ParserRule[] = []): ParserRule { + return ModelRule; +} export const ModelTerminals = [WS, HIDDEN_NEWLINE, ML_COMMENT, SL_COMMENT]; From 42b8c79d9e7db9fcf7acd783b211ccc369003b61 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:42:25 -0400 Subject: [PATCH 19/41] Fix scope resolution for CsvClassImport class references --- .../language-model/src/features/modelScopeProvider.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/packages/language-model/src/features/modelScopeProvider.ts b/app/packages/language-model/src/features/modelScopeProvider.ts index 42eeda12..47a40fb6 100644 --- a/app/packages/language-model/src/features/modelScopeProvider.ts +++ b/app/packages/language-model/src/features/modelScopeProvider.ts @@ -3,11 +3,13 @@ import { sharedImport, resolveRelativePath } from "@mdeo/language-shared"; import type { AstNodeDescriptionProvider, LangiumDocuments, ReferenceInfo, Scope } from "langium"; import { ObjectInstance, + CsvClassImport, PropertyAssignment, EnumValue, LinkEnd, type ModelType, type ObjectInstanceType, + type CsvClassImportType, type PropertyAssignmentType, type LinkEndType, type EnumValueType, @@ -72,7 +74,7 @@ export class ModelScopeProvider extends DefaultScopeProvider { override getScope(context: ReferenceInfo): Scope { const document = AstUtils.getDocument(context.container); - if (context.property === "class" && this.astReflection.isInstance(context.container, ObjectInstance)) { + if (context.property === "class" && (this.astReflection.isInstance(context.container, ObjectInstance) || this.astReflection.isInstance(context.container, CsvClassImport))) { return this.getObjectClassScope(context, document); } if (context.property === "name" && this.astReflection.isInstance(context.container, PropertyAssignment)) { From bf4f83ff46cbddcc062571d193b423faa0f4f63a Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:51:16 -0400 Subject: [PATCH 20/41] Fix scope resolution for CsvClassImport: traverse two levels up to reach Model --- .../language-model/src/features/modelScopeProvider.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/packages/language-model/src/features/modelScopeProvider.ts b/app/packages/language-model/src/features/modelScopeProvider.ts index 47a40fb6..e9754963 100644 --- a/app/packages/language-model/src/features/modelScopeProvider.ts +++ b/app/packages/language-model/src/features/modelScopeProvider.ts @@ -105,8 +105,9 @@ export class ModelScopeProvider extends DefaultScopeProvider { * @returns A scope containing all accessible classes from the imported metamodel */ private getObjectClassScope(context: ReferenceInfo, document: any): Scope { - const model = context.container.$container as ModelType; - const metamodelImport = model.import; + const parent = context.container.$container; + const model = (this.astReflection.isInstance(parent, Model) ? parent : parent?.$container) as ModelType; + const metamodelImport = model?.import; const relativePath = metamodelImport?.file; if (relativePath == undefined) { From 75b673c0a285c76b964953605cf2ba35a38114a1 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 15:59:14 -0400 Subject: [PATCH 21/41] Register modelDataHandler in service-model for CSV import block support --- app/packages/service-model/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index 99f448af..f72ad188 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -44,6 +44,7 @@ const modelLanguagePlugin: LanguagePlugin = { initializePluginContext(); const { modelPluginProvider, registerModelContributionPlugin } = await import("@mdeo/language-model"); +const { modelDataHandler, MODEL_DATA_HANDLER_KEY } = await import("./handler/modelDataHandler.js"); const { createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); registerModelContributionPlugin(createModelCsvContributionPlugin()); @@ -63,7 +64,8 @@ const modelLanguageConfig: LanguageServiceConfig = { languagePlugin: modelLanguagePlugin, languagePluginProvider: modelPluginProvider, fileDataHandlers: { - [AST_HANDLER_KEY]: astHandler + [AST_HANDLER_KEY]: astHandler, + [MODEL_DATA_HANDLER_KEY]: modelDataHandler } }; From e1a780447e79be093bddf605b721b238d89ca07a Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:33:21 -0400 Subject: [PATCH 22/41] Read CSV files in GModelFactory to render diagram nodes from import CSV block --- .../diagram-server/modelGModelFactory.ts | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index b0d32f39..156a3852 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -56,7 +56,7 @@ export class ModelGModelFactory extends BaseGModelFactory { override async createModelInternal(sourceModel: PartialModel, idRegistry: ModelIdRegistry): Promise { const graph = GGraph.builder().id("model-graph").addCssClass("editor-model").build(); - const extracted = this.extractElements(sourceModel); + const extracted = await this.extractElements(sourceModel); await this.createObjectNodes(graph, extracted.objects, idRegistry); await this.createLinkEdges(graph, extracted.links, idRegistry); return graph; @@ -68,10 +68,10 @@ export class ModelGModelFactory extends BaseGModelFactory { * @param model The model containing elements * @returns An object containing separate arrays of objects and links */ - private extractElements(model: PartialModel): { + private async extractElements(model: PartialModel): Promise<{ objects: PartialObjectInstance[]; links: PartialLink[]; - } { + }> { const objects: PartialObjectInstance[] = []; const links: PartialLink[] = []; @@ -87,6 +87,29 @@ export class ModelGModelFactory extends BaseGModelFactory { } } + if (model.csvImport != undefined) { + for (const entry of model.csvImport.imports ?? []) { + const classRef = entry.class?.ref; + if (classRef == undefined) continue; + try { + const doc = this.modelState.sourceModel.$document; + if (!doc) continue; + const uri = resolveRelativePath(doc, entry.file ?? ""); + const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); + const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); + objects.push({ + $type: "ObjectInstance", + name: `${classRef.name}_${rowIndex}`, + class: { ref: classRef, $refText: classRef.name }, + properties: [] + } as unknown as PartialObjectInstance); + }); + } catch { + // CSV file not readable, skip + } + } + } + return { objects, links }; } From 5dd26ccf80cbfe7d15314f05f294d1c5a81b3924 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:44:43 -0400 Subject: [PATCH 23/41] Fix modelGModelFactory: add CSV import block rendering with FileSystemProvider --- .../src/features/diagram-server/modelGModelFactory.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 156a3852..3bfce7dc 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -6,7 +6,8 @@ import { GHorizontalDivider, EdgeLayoutMetadataUtil, NodeLayoutMetadataUtil, - parseIdentifier + parseIdentifier, + resolveRelativePath } from "@mdeo/language-shared"; import type { ModelIdRegistry, GraphMetadata } from "@mdeo/language-shared"; import type { NodeLayoutMetadata, EdgeLayoutMetadata } from "@mdeo/protocol-common"; @@ -92,11 +93,13 @@ export class ModelGModelFactory extends BaseGModelFactory { const classRef = entry.class?.ref; if (classRef == undefined) continue; try { - const doc = this.modelState.sourceModel.$document; + const doc = this.modelState.sourceModel?.$document; if (!doc) continue; const uri = resolveRelativePath(doc, entry.file ?? ""); const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); + if (rows.length < 2) continue; + rows.slice(1).forEach((_: string[], rowIndex: number) => { objects.push({ $type: "ObjectInstance", name: `${classRef.name}_${rowIndex}`, From 4aa65d901b101355c2b4867c48dab4071c2cb9a2 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 11 Jul 2026 23:54:02 -0400 Subject: [PATCH 24/41] Add createCsvNodes to render diagram nodes directly from CSV import block --- .../diagram-server/modelGModelFactory.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 3bfce7dc..29765cdb 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -60,6 +60,7 @@ export class ModelGModelFactory extends BaseGModelFactory { const extracted = await this.extractElements(sourceModel); await this.createObjectNodes(graph, extracted.objects, idRegistry); await this.createLinkEdges(graph, extracted.links, idRegistry); + await this.createCsvNodes(graph, sourceModel); return graph; } @@ -487,4 +488,38 @@ export class ModelGModelFactory extends BaseGModelFactory { return nodes; } + private async createCsvNodes(graph: GGraphType, model: PartialModel): Promise { + if (model.csvImport == undefined) return; + const doc = this.modelState.sourceModel?.$document; + if (!doc) return; + + let nodeIndex = 0; + for (const entry of model.csvImport.imports ?? []) { + const classRef = entry.class?.ref; + if (classRef == undefined) continue; + try { + const uri = resolveRelativePath(doc, entry.file ?? ""); + const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); + const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); + if (rows.length < 2) continue; + rows.slice(1).forEach((row: string[], rowIndex: number) => { + const instanceName = `${classRef.name}_${rowIndex}`; + const nodeId = `csv-node-${nodeIndex++}`; + const headerComp = GCompartment.builder() + .id(`${nodeId}__header`) + .addCssClass("object-header") + .add(GObjectNameLabel.builder().id(`${nodeId}__name`).text(`${instanceName} : ${classRef.name}`).build()) + .build(); + const node = GObjectNode.builder() + .id(nodeId) + .add(headerComp) + .build(); + graph.children.push(node); + }); + } catch { + // skip unreadable CSV + } + } + } + } From 286ac906a1a94312e41b801fa4582aea32afa5ed Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:16:44 -0400 Subject: [PATCH 25/41] Remove synthetic PartialObjectInstance from extractElements, use createCsvNodes instead --- .../diagram-server/modelGModelFactory.ts | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 29765cdb..6a05f2bd 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -89,31 +89,6 @@ export class ModelGModelFactory extends BaseGModelFactory { } } - if (model.csvImport != undefined) { - for (const entry of model.csvImport.imports ?? []) { - const classRef = entry.class?.ref; - if (classRef == undefined) continue; - try { - const doc = this.modelState.sourceModel?.$document; - if (!doc) continue; - const uri = resolveRelativePath(doc, entry.file ?? ""); - const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); - const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); - if (rows.length < 2) continue; - rows.slice(1).forEach((_: string[], rowIndex: number) => { - objects.push({ - $type: "ObjectInstance", - name: `${classRef.name}_${rowIndex}`, - class: { ref: classRef, $refText: classRef.name }, - properties: [] - } as unknown as PartialObjectInstance); - }); - } catch { - // CSV file not readable, skip - } - } - } - return { objects, links }; } From 5cbe9ffc647ec81f358dbe84025a07ba736e7e47 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:23:34 -0400 Subject: [PATCH 26/41] Fix createCsvNodes: use proper GObjectNode builder with metadata and class hierarchy --- .../diagram-server/modelGModelFactory.ts | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 6a05f2bd..f7fd07f8 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -468,27 +468,30 @@ export class ModelGModelFactory extends BaseGModelFactory { const doc = this.modelState.sourceModel?.$document; if (!doc) return; + const validatedMetadata = await this.modelState.getValidatedMetadata(); + let nodeIndex = 0; for (const entry of model.csvImport.imports ?? []) { - const classRef = entry.class?.ref; + const classRef = entry.class?.ref as ClassType | undefined; if (classRef == undefined) continue; try { const uri = resolveRelativePath(doc, entry.file ?? ""); const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); if (rows.length < 2) continue; - rows.slice(1).forEach((row: string[], rowIndex: number) => { + const classHierarchy = resolveClassChain(classRef, this.reflection).map((c) => c.name); + rows.slice(1).forEach((_row: string[], rowIndex: number) => { const instanceName = `${classRef.name}_${rowIndex}`; const nodeId = `csv-node-${nodeIndex++}`; - const headerComp = GCompartment.builder() - .id(`${nodeId}__header`) - .addCssClass("object-header") - .add(GObjectNameLabel.builder().id(`${nodeId}__name`).text(`${instanceName} : ${classRef.name}`).build()) - .build(); + const metadata = validatedMetadata.nodes[nodeId]?.meta ?? {}; const node = GObjectNode.builder() .id(nodeId) - .add(headerComp) + .name(instanceName) + .typeName(classRef.name) + .classHierarchy(classHierarchy) + .meta(metadata) .build(); + node.children.push(...this.createObjectHeader(nodeId, instanceName, classRef.name)); graph.children.push(node); }); } catch { From 67feb390c4be123bf9cdf1753c98e0ee9de6dd18 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 12 Jul 2026 00:48:34 -0400 Subject: [PATCH 27/41] Persist CSV node positions by carrying forward metadata from previous render --- .../diagram-server/modelMetadataManager.ts | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts index b83bd46d..b5bf73c2 100644 --- a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts +++ b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts @@ -63,10 +63,32 @@ export class ModelMetadataManager extends MetadataManager { this.extractObjectMetadata(objects, idRegistry, nodes); this.extractLinkMetadata(links, idRegistry, edges, nodes); + this.extractCsvNodeMetadata(sourceModel, nodes); return { nodes, edges }; } + private storedCurrentMetadata: GraphMetadata | undefined; + + override async validateMetadata( + sourceModel: PartialModel, + currentMetadata: GraphMetadata, + lastValidMetadata: GraphMetadata + ): Promise { + this.storedCurrentMetadata = currentMetadata; + return super.validateMetadata(sourceModel, currentMetadata, lastValidMetadata); + } + + private extractCsvNodeMetadata(sourceModel: PartialModel, nodes: Record): void { + if (sourceModel.csvImport == undefined) return; + if (this.storedCurrentMetadata == undefined) return; + for (const [id, meta] of Object.entries(this.storedCurrentMetadata.nodes)) { + if (id.startsWith("csv-node-") && !nodes[id]) { + nodes[id] = meta; + } + } + } + /** * Extracts objects and links from the model. * From f4a61011f2cb6de459987e748f9812262c7e6674 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Mon, 13 Jul 2026 21:35:24 -0400 Subject: [PATCH 28/41] Remove CSV-to-model materialize endpoint and dead inference engine The POST /api/projects/{id}/csv-import endpoint materialized a CSV into a standalone .metamodel/.model file pair using metamodel inference. That workflow is superseded by the import CSV {} syntax, which reads CSV data directly at render/execution time against an existing, manually authored metamodel. Removes CsvImportService, CsvImportRoutes, the now-unused CsvModelInference engine and its tests, the backend's :metamodel dependency (no longer needed), and the unused CSV_IMPORT_FAILED error code. --- platform/backend/build.gradle.kts | 3 +- .../kotlin/com/mdeo/backend/Application.kt | 2 - .../mdeo/backend/routes/CsvImportRoutes.kt | 71 ------ .../mdeo/backend/service/CsvImportService.kt | 221 ------------------ .../mdeo/backend/service/InjectedServices.kt | 5 - .../kotlin/com/mdeo/common/model/ApiResult.kt | 4 +- .../mdeo/metamodel/csv/CsvModelInference.kt | 191 --------------- .../csv/CsvModelInferenceManualRun.kt | 41 ---- .../metamodel/csv/CsvModelInferenceTest.kt | 158 ------------- 9 files changed, 2 insertions(+), 694 deletions(-) delete mode 100644 platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt delete mode 100644 platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt delete mode 100644 platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt delete mode 100644 platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt delete mode 100644 platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt diff --git a/platform/backend/build.gradle.kts b/platform/backend/build.gradle.kts index a2aff488..0b0b6990 100644 --- a/platform/backend/build.gradle.kts +++ b/platform/backend/build.gradle.kts @@ -11,8 +11,7 @@ application { dependencies { implementation(project(":common")) - implementation(project(":metamodel")) - + // Ktor Server implementation(libs.ktor.server.core) implementation(libs.ktor.server.netty) diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/Application.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/Application.kt index eae0234e..4ba78337 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/Application.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/Application.kt @@ -59,7 +59,6 @@ fun Application.module(appConfig: AppConfig) { override val executionService: ExecutionService by lazy { ExecutionService(this) } override val webSocketNotificationService: WebSocketNotificationService by lazy { WebSocketNotificationService() } override val languagePluginRequestService: LanguagePluginRequestService by lazy { LanguagePluginRequestService(this) } - override val csvImportService: CsvImportService by lazy { CsvImportService(this) } } services.jwtService.init() @@ -123,7 +122,6 @@ fun Application.module(appConfig: AppConfig) { adminRoutes(services.userService) userRoutes(services.userService, services.projectService) executionRoutes(services.executionService, services.projectService) - csvImportRoutes(services.csvImportService, services.projectService) } } } diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt deleted file mode 100644 index 06a82de9..00000000 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/routes/CsvImportRoutes.kt +++ /dev/null @@ -1,71 +0,0 @@ -package com.mdeo.backend.routes - -import com.mdeo.backend.plugins.* -import com.mdeo.backend.service.CsvImportService -import com.mdeo.backend.service.ProjectPermission -import com.mdeo.backend.service.ProjectService -import io.ktor.http.* -import io.ktor.server.request.* -import io.ktor.server.response.* -import io.ktor.server.routing.* -import java.util.* - -fun Route.csvImportRoutes( - csvImportService: CsvImportService, - projectService: ProjectService -) { - route("/api/projects/{projectId}/csv-import") { - post { - val session = call.getUserSession() - if (session == null) { - call.respond(HttpStatusCode.Unauthorized) - return@post - } - - val projectId = call.parameters["projectId"]?.let { - try { UUID.fromString(it) } catch (e: Exception) { null } - } - if (projectId == null) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Invalid project ID")) - return@post - } - - val userId = try { UUID.fromString(session.userId) } catch (e: Exception) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Invalid user ID")) - return@post - } - - if (!projectService.hasProjectPermission(projectId, userId, call.isAdmin(), ProjectPermission.WRITE)) { - call.respond(HttpStatusCode.Forbidden, mapOf("error" to "Access denied")) - return@post - } - - val basePath = call.request.queryParameters["path"] - if (basePath.isNullOrBlank()) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Missing path parameter")) - return@post - } - - val metamodelPath = call.request.queryParameters["metamodelPath"] - if (metamodelPath.isNullOrBlank()) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Missing metamodelPath parameter")) - return@post - } - - val className = call.request.queryParameters["className"] - if (className.isNullOrBlank()) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Missing className parameter")) - return@post - } - - val csvText = call.receiveText() - if (csvText.isBlank()) { - call.respond(HttpStatusCode.BadRequest, mapOf("error" to "Request body is empty")) - return@post - } - - val result = csvImportService.importCsv(projectId, csvText, basePath, className, metamodelPath) - call.respondApiResult(result) - } - } -} diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt deleted file mode 100644 index dfeddd3a..00000000 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/CsvImportService.kt +++ /dev/null @@ -1,221 +0,0 @@ -package com.mdeo.backend.service - -import com.mdeo.common.model.ApiResult -import com.mdeo.common.model.ErrorCodes -import com.mdeo.common.model.commonFailure -import com.mdeo.common.model.success -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.* -import java.util.UUID - -@Serializable -data class CsvImportResult( - val modelPath: String, - val instanceCount: Int, - val warnings: List -) - -class CsvImportService(services: InjectedServices) : BaseService(), InjectedServices by services { - - private val json = Json { ignoreUnknownKeys = true; prettyPrint = true } - - suspend fun importCsv( - projectId: UUID, - csvText: String, - basePath: String, - className: String, - metamodelPath: String - ): ApiResult { - val normalizedBasePath = basePath.trim().trim('/') - if (normalizedBasePath.isBlank()) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Invalid path parameter") - } - val modelPath = "$normalizedBasePath.m" - - val metamodelDataResult = fileDataService.getFileData(projectId, metamodelPath, null, "metamodel") - if (metamodelDataResult is ApiResult.Failure) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not compute metamodel data for '$metamodelPath'") - } - val metamodelJson = (metamodelDataResult as ApiResult.Success).value.data - if (metamodelJson == kotlinx.serialization.json.JsonNull) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Metamodel '$metamodelPath' has errors or could not be parsed") - } - - val metamodelData = try { - json.decodeFromJsonElement(metamodelJson) - } catch (e: Exception) { - return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "Could not parse metamodel data: ${e.message}") - } - - val targetClass = metamodelData.classes.find { it.name == className } - ?: return commonFailure( - ErrorCodes.CSV_IMPORT_FAILED, - "Class '$className' not found in metamodel. Available: ${metamodelData.classes.map { it.name }}" - ) - - val warnings = mutableListOf() - - val rows = parseCsv(csvText) - if (rows.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV has no header row") - val header = rows.first() - if (header.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV header row is empty") - val dataRows = rows.drop(1) - if (dataRows.isEmpty()) return commonFailure(ErrorCodes.CSV_IMPORT_FAILED, "CSV has no data rows") - - val propertyNames = targetClass.properties.map { it.name }.toSet() - header.forEach { col -> - if (col != "_id" && col !in propertyNames) { - warnings.add("Column '$col' does not match any property in class '$className' — it will be ignored.") - } - } - - val missingRequired = targetClass.properties.filter { prop -> - prop.multiplicity.lower > 0 && header.none { it == prop.name } - } - if (missingRequired.isNotEmpty()) { - return commonFailure( - ErrorCodes.CSV_IMPORT_FAILED, - "CSV is missing required properties: ${missingRequired.map { it.name }}" - ) - } - - val modelFileDepth = normalizedBasePath.count { it == '/' } - val relativePrefix = "../".repeat(modelFileDepth) - val relativeMetamodelPath = "$relativePrefix$metamodelPath" - - val dslLines = StringBuilder() - dslLines.appendLine("""using "$relativeMetamodelPath"""") - dslLines.appendLine() - - dataRows.forEachIndexed { rowIndex, row -> - val normalizedRow = normalizeRow(row, header.size, rowIndex + 2, warnings) - val instanceName = "${className}_$rowIndex" - dslLines.appendLine("$instanceName : $className {") - header.forEachIndexed { colIndex, colName -> - if (colName == "_id") return@forEachIndexed - val prop = targetClass.properties.find { it.name == colName } ?: return@forEachIndexed - val rawValue = normalizedRow[colIndex] - if (rawValue.isBlank()) return@forEachIndexed - val formattedValue = formatDslValue(rawValue, prop) - dslLines.appendLine(" $colName = $formattedValue") - } - dslLines.appendLine("}") - dslLines.appendLine() - } - - val modelBytes = dslLines.toString().toByteArray(Charsets.UTF_8) - - when (val result = fileService.writeFile(projectId, modelPath, modelBytes, create = true, overwrite = true)) { - is ApiResult.Failure -> return ApiResult.Failure(result.error) - is ApiResult.Success -> {} - } - - return success( - CsvImportResult( - modelPath = modelPath, - instanceCount = dataRows.size, - warnings = warnings - ) - ) - } - - private fun formatDslValue(rawValue: String, prop: PropertyData): String { - return when { - prop.enumType != null -> rawValue - prop.primitiveType == "int" || prop.primitiveType == "long" -> - rawValue.toLongOrNull()?.toString() ?: "\"$rawValue\"" - prop.primitiveType == "double" || prop.primitiveType == "float" -> - rawValue.toDoubleOrNull()?.toString() ?: "\"$rawValue\"" - prop.primitiveType == "boolean" -> - if (rawValue.equals("true", ignoreCase = true)) "true" else "false" - else -> "\"${rawValue.replace("\\", "\\\\").replace("\"", "\\\"")}\"" - } - } - - private fun normalizeRow(row: List, expectedSize: Int, rowNumber: Int, warnings: MutableList): List { - return when { - row.size == expectedSize -> row - row.size < expectedSize -> { - warnings.add("Row $rowNumber has fewer columns than the header; missing values treated as blank.") - row + List(expectedSize - row.size) { "" } - } - else -> { - warnings.add("Row $rowNumber has more columns than the header; extra values ignored.") - row.take(expectedSize) - } - } - } - - private fun parseCsv(text: String): List> { - val rows = mutableListOf>() - var currentRow = mutableListOf() - val currentField = StringBuilder() - var inQuotes = false - var i = 0 - - fun endField() { currentRow.add(currentField.toString()); currentField.clear() } - fun endRow() { - endField() - if (!(currentRow.size == 1 && currentRow[0].isBlank())) rows.add(currentRow) - currentRow = mutableListOf() - } - - while (i < text.length) { - val c = text[i] - if (inQuotes) { - when (c) { - '"' -> if (i + 1 < text.length && text[i + 1] == '"') { currentField.append('"'); i++ } - else inQuotes = false - else -> currentField.append(c) - } - } else { - when (c) { - '"' -> inQuotes = true - ',' -> endField() - '\r' -> {} - '\n' -> endRow() - else -> currentField.append(c) - } - } - i++ - } - if (currentField.isNotEmpty() || currentRow.isNotEmpty()) endRow() - return rows - } - - @Serializable - private data class MultiplicityData(val lower: Int, val upper: Int) - - @Serializable - private data class PropertyData( - val name: String, - val enumType: String? = null, - val primitiveType: String? = null, - val multiplicity: MultiplicityData - ) - - @Serializable - private data class ClassData( - val name: String, - val isAbstract: Boolean, - val properties: List - ) - - @Serializable - private data class EnumData(val name: String, val entries: List) - - @Serializable - private data class AssociationEndData(val className: String, val name: String? = null, val multiplicity: MultiplicityData) - - @Serializable - private data class AssociationData(val source: AssociationEndData, val operator: String, val target: AssociationEndData) - - @Serializable - private data class MetamodelAstData( - val path: String, - val classes: List, - val enums: List, - val associations: List, - val importedMetamodelPaths: List = emptyList() - ) -} diff --git a/platform/backend/src/main/kotlin/com/mdeo/backend/service/InjectedServices.kt b/platform/backend/src/main/kotlin/com/mdeo/backend/service/InjectedServices.kt index 5fe30fae..8c97090f 100644 --- a/platform/backend/src/main/kotlin/com/mdeo/backend/service/InjectedServices.kt +++ b/platform/backend/src/main/kotlin/com/mdeo/backend/service/InjectedServices.kt @@ -67,9 +67,4 @@ interface InjectedServices { * Service for executing language plugin requests. */ val languagePluginRequestService: LanguagePluginRequestService - - /** - * Service for importing CSV files as inferred metamodel/model pairs. - */ - val csvImportService: CsvImportService } diff --git a/platform/common/src/main/kotlin/com/mdeo/common/model/ApiResult.kt b/platform/common/src/main/kotlin/com/mdeo/common/model/ApiResult.kt index 53961c0e..b7a67bc1 100644 --- a/platform/common/src/main/kotlin/com/mdeo/common/model/ApiResult.kt +++ b/platform/common/src/main/kotlin/com/mdeo/common/model/ApiResult.kt @@ -66,9 +66,7 @@ object ErrorCodes { const val FILE_DATA_CIRCULAR_DEPENDENCY = "FileDataCircularDependency" const val FILE_DATA_COMPUTATION_FAILED = "FileDataComputationFailed" const val FILE_DATA_NO_PLUGIN_FOUND = "FileDataNoPluginFound" - - const val CSV_IMPORT_FAILED = "CsvImportFailed" - + const val EXECUTION_NOT_FOUND = "ExecutionNotFound" const val EXECUTION_ALREADY_COMPLETED = "ExecutionAlreadyCompleted" const val EXECUTION_INVALID_STATE = "ExecutionInvalidState" diff --git a/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt b/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt deleted file mode 100644 index 5a2fd1d2..00000000 --- a/platform/metamodel/src/main/kotlin/com/mdeo/metamodel/csv/CsvModelInference.kt +++ /dev/null @@ -1,191 +0,0 @@ -package com.mdeo.metamodel.csv - -import com.mdeo.metamodel.data.ClassData -import com.mdeo.metamodel.data.MetamodelData -import com.mdeo.metamodel.data.ModelData -import com.mdeo.metamodel.data.ModelDataInstance -import com.mdeo.metamodel.data.ModelDataPropertyValue - -object CsvModelInference { - - data class ImportResult( - val model: ModelData, - val warnings: List - ) - - fun importFromCsv( - csvText: String, - className: String, - metamodel: MetamodelData, - metamodelPath: String - ): ImportResult { - val targetClass = metamodel.classes.find { it.name == className } - ?: throw IllegalArgumentException( - "Class '$className' not found in metamodel. Available classes: ${metamodel.classes.map { it.name }.joinToString()}" - ) - - val rows = parseCsv(csvText) - require(rows.isNotEmpty()) { "CSV has no header row" } - - val header = rows.first() - require(header.isNotEmpty()) { "CSV header row is empty" } - - val dataRows = rows.drop(1) - require(dataRows.isNotEmpty()) { "CSV has a header row but no data rows" } - - val warnings = mutableListOf() - - validateHeaders(header, targetClass, warnings) - - val normalizedRows = dataRows.mapIndexed { rowIndex, row -> - when { - row.size == header.size -> row - row.size < header.size -> { - warnings.add("Row ${rowIndex + 2} has fewer columns than the header; missing values treated as blank.") - row + List(header.size - row.size) { "" } - } - else -> { - warnings.add("Row ${rowIndex + 2} has more columns than the header; extra values ignored.") - row.take(header.size) - } - } - } - - val instances = normalizedRows.mapIndexed { rowIndex, row -> - val instanceProperties = header.indices.associate { colIndex -> - val columnName = header[colIndex].trim() - val property = targetClass.properties.find { it.name == columnName } - val rawValue = row[colIndex] - columnName to cellToPropertyValue(rawValue, property, metamodel) - } - ModelDataInstance( - name = "${className}_$rowIndex", - className = className, - properties = instanceProperties - ) - } - - val model = ModelData( - metamodelPath = metamodelPath, - instances = instances, - links = emptyList() - ) - - return ImportResult(model = model, warnings = warnings) - } - - private fun validateHeaders( - header: List, - targetClass: ClassData, - warnings: MutableList - ) { - val propertyNames = targetClass.properties.map { it.name }.toSet() - - header.forEach { columnName -> - val trimmed = columnName.trim() - if (trimmed !in propertyNames) { - warnings.add("Column '$trimmed' does not match any property in class '${targetClass.name}' and will be stored as-is.") - } - } - - val missingRequired = targetClass.properties.filter { property -> - property.multiplicity.isRequired() && header.none { it.trim() == property.name } - } - if (missingRequired.isNotEmpty()) { - throw IllegalArgumentException( - "CSV is missing required properties for class '${targetClass.name}': ${missingRequired.map { it.name }.joinToString()}" - ) - } - } - - private fun cellToPropertyValue( - rawValue: String, - property: com.mdeo.metamodel.data.PropertyData?, - metamodel: MetamodelData - ): ModelDataPropertyValue { - if (rawValue.isBlank()) { - return ModelDataPropertyValue.NullValue - } - - if (property == null) { - return ModelDataPropertyValue.StringValue(rawValue) - } - - return when { - property.enumType != null -> { - val enumDef = metamodel.enums.find { it.name == property.enumType } - if (enumDef != null && rawValue !in enumDef.entries) { - ModelDataPropertyValue.StringValue(rawValue) - } else { - ModelDataPropertyValue.EnumValue(rawValue) - } - } - property.primitiveType == "int" || property.primitiveType == "long" -> { - rawValue.toLongOrNull()?.let { ModelDataPropertyValue.NumberValue(it.toDouble()) } - ?: ModelDataPropertyValue.StringValue(rawValue) - } - property.primitiveType == "double" || property.primitiveType == "float" -> { - rawValue.toDoubleOrNull()?.let { ModelDataPropertyValue.NumberValue(it) } - ?: ModelDataPropertyValue.StringValue(rawValue) - } - property.primitiveType == "boolean" -> { - ModelDataPropertyValue.BooleanValue(rawValue.equals("true", ignoreCase = true)) - } - else -> ModelDataPropertyValue.StringValue(rawValue) - } - } - - private fun parseCsv(text: String): List> { - val rows = mutableListOf>() - var currentRow = mutableListOf() - val currentField = StringBuilder() - var inQuotes = false - var i = 0 - val n = text.length - - fun endField() { - currentRow.add(currentField.toString()) - currentField.clear() - } - - fun endRow() { - endField() - if (!(currentRow.size == 1 && currentRow[0].isBlank())) { - rows.add(currentRow) - } - currentRow = mutableListOf() - } - - while (i < n) { - val c = text[i] - if (inQuotes) { - when (c) { - '"' -> { - if (i + 1 < n && text[i + 1] == '"') { - currentField.append('"') - i++ - } else { - inQuotes = false - } - } - else -> currentField.append(c) - } - } else { - when (c) { - '"' -> inQuotes = true - ',' -> endField() - '\r' -> {} - '\n' -> endRow() - else -> currentField.append(c) - } - } - i++ - } - - if (currentField.isNotEmpty() || currentRow.isNotEmpty()) { - endRow() - } - - return rows - } -} diff --git a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt deleted file mode 100644 index 28470de5..00000000 --- a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceManualRun.kt +++ /dev/null @@ -1,41 +0,0 @@ -package com.mdeo.metamodel.csv - -import com.mdeo.metamodel.data.MetamodelData -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import org.junit.jupiter.api.Disabled -import org.junit.jupiter.api.Test -import java.nio.file.Path - -class CsvModelInferenceManualRun { - - @Test - @Disabled("Manual run only (requires local CSV and metamodel files).") - fun `import csv onto existing metamodel`() { - val csvPath = Path.of("/Users/amirguliyev/Downloads/employees.csv") - val metamodelPath = Path.of("/Users/amirguliyev/Downloads/employees.metamodel") - - val csvText = csvPath.toFile().readText() - val metamodel = Json.decodeFromString(metamodelPath.toFile().readText()) - - val className = csvPath.fileName.toString().removeSuffix(".csv") - .replaceFirstChar { it.uppercaseChar() } - - val result = CsvModelInference.importFromCsv( - csvText = csvText, - className = className, - metamodel = metamodel, - metamodelPath = metamodelPath.toString() - ) - - val json = Json { prettyPrint = true } - - println("===== MODEL =====") - println(json.encodeToString(result.model)) - - if (result.warnings.isNotEmpty()) { - println("===== WARNINGS =====") - result.warnings.forEach { println(it) } - } - } -} diff --git a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt b/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt deleted file mode 100644 index d4de7ddd..00000000 --- a/platform/metamodel/src/test/kotlin/com/mdeo/metamodel/csv/CsvModelInferenceTest.kt +++ /dev/null @@ -1,158 +0,0 @@ -package com.mdeo.metamodel.csv - -import com.mdeo.metamodel.data.ClassData -import com.mdeo.metamodel.data.EnumData -import com.mdeo.metamodel.data.MetamodelData -import com.mdeo.metamodel.data.ModelDataPropertyValue -import com.mdeo.metamodel.data.MultiplicityData -import com.mdeo.metamodel.data.PropertyData -import org.junit.jupiter.api.Assertions.* -import org.junit.jupiter.api.Test - -class CsvModelInferenceTest { - - private fun metamodel(vararg classes: ClassData, enums: List = emptyList()) = - MetamodelData(path = "/test.metamodel", classes = classes.toList(), enums = enums) - - private fun cls(name: String, vararg properties: PropertyData) = - ClassData(name = name, isAbstract = false, properties = properties.toList()) - - private fun prop(name: String, primitiveType: String, required: Boolean = true) = - PropertyData( - name = name, - primitiveType = primitiveType, - enumType = null, - multiplicity = if (required) MultiplicityData.single() else MultiplicityData.optional() - ) - - private fun enumProp(name: String, enumType: String, required: Boolean = true) = - PropertyData( - name = name, - primitiveType = null, - enumType = enumType, - multiplicity = if (required) MultiplicityData.single() else MultiplicityData.optional() - ) - - @Test - fun `maps columns to metamodel properties by name`() { - val mm = metamodel(cls("Employee", - prop("id", "int"), - prop("name", "string"), - prop("salary", "double"), - prop("active", "boolean") - )) - - val csv = """ - id,name,salary,active - 1,Alice,95000.0,true - 2,Bob,72000.5,false - """.trimIndent() - - val result = CsvModelInference.importFromCsv(csv, "Employee", mm, "/test.metamodel") - - assertEquals(2, result.model.instances.size) - val alice = result.model.instances[0] - assertEquals("Employee_0", alice.name) - assertEquals(ModelDataPropertyValue.NumberValue(1.0), alice.properties["id"]) - assertEquals(ModelDataPropertyValue.StringValue("Alice"), alice.properties["name"]) - assertEquals(ModelDataPropertyValue.NumberValue(95000.0), alice.properties["salary"]) - assertEquals(ModelDataPropertyValue.BooleanValue(true), alice.properties["active"]) - } - - @Test - fun `maps enum values from metamodel`() { - val mm = metamodel( - cls("Ticket", enumProp("status", "StatusEnum")), - enums = listOf(EnumData(name = "StatusEnum", entries = listOf("OPEN", "CLOSED"))) - ) - - val csv = "status\nOPEN\nCLOSED\nOPEN" - val result = CsvModelInference.importFromCsv(csv, "Ticket", mm, "/test.metamodel") - - assertEquals(ModelDataPropertyValue.EnumValue("OPEN"), result.model.instances[0].properties["status"]) - assertEquals(ModelDataPropertyValue.EnumValue("CLOSED"), result.model.instances[1].properties["status"]) - } - - @Test - fun `treats blank cells as null`() { - val mm = metamodel(cls("Person", - prop("name", "string"), - prop("nickname", "string", required = false) - )) - - val csv = "name,nickname\nAlice,Ace\nBob," - val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") - - assertEquals(ModelDataPropertyValue.NullValue, result.model.instances[1].properties["nickname"]) - } - - @Test - fun `warns about unknown columns`() { - val mm = metamodel(cls("Person", prop("name", "string"))) - val csv = "name,unknown_col\nAlice,extra" - val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") - - assertTrue(result.warnings.any { it.contains("unknown_col") }) - } - - @Test - fun `throws when class not found in metamodel`() { - val mm = metamodel(cls("Person", prop("name", "string"))) - val csv = "name\nAlice" - - assertThrows(IllegalArgumentException::class.java) { - CsvModelInference.importFromCsv(csv, "NonExistent", mm, "/test.metamodel") - } - } - - @Test - fun `throws when required property is missing from csv`() { - val mm = metamodel(cls("Person", - prop("name", "string"), - prop("id", "int") - )) - val csv = "name\nAlice" - - assertThrows(IllegalArgumentException::class.java) { - CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") - } - } - - @Test - fun `pads ragged short rows and warns`() { - val mm = metamodel(cls("Person", - prop("name", "string"), - prop("age", "int", required = false) - )) - val csv = "name,age\nAlice,30\nBob" - val result = CsvModelInference.importFromCsv(csv, "Person", mm, "/test.metamodel") - - assertEquals(2, result.model.instances.size) - assertEquals(ModelDataPropertyValue.NullValue, result.model.instances[1].properties["age"]) - assertTrue(result.warnings.any { it.contains("fewer columns") }) - } - - @Test - fun `handles quoted fields with embedded commas`() { - val mm = metamodel(cls("Item", prop("description", "string"))) - val csv = "description\n\"Contains, a comma\"" - val result = CsvModelInference.importFromCsv(csv, "Item", mm, "/test.metamodel") - - assertEquals( - ModelDataPropertyValue.StringValue("Contains, a comma"), - result.model.instances[0].properties["description"] - ) - } - - @Test - fun `gracefully falls back to string for unparseable int`() { - val mm = metamodel(cls("Item", prop("count", "int"))) - val csv = "count\nnot_a_number" - val result = CsvModelInference.importFromCsv(csv, "Item", mm, "/test.metamodel") - - assertEquals( - ModelDataPropertyValue.StringValue("not_a_number"), - result.model.instances[0].properties["count"] - ) - } -} From 640ba1f9dab1a3235fba1158a9dcc324261b0fde Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Tue, 14 Jul 2026 00:20:39 -0400 Subject: [PATCH 29/41] Turn CSV import into a proper contribution plugin The import CSV {} syntax was hardcoded directly into the base model grammar. Model now exposes a generic extension point (imports: [BaseModelImport], mirroring Config's sections: [BaseConfigSection]), and CSV is implemented as a real contribution plugin using the same SerializedGrammar-based merging approach as config/config-mdeo: - ModelContributionPlugin now carries a serialized grammar and import metadata instead of raw parser rule objects, so it can cross service boundaries as plain JSON - resolveModelPlugins deserializes and wraps each contributed import in a BaseModelImport-extending rule, and createModelRule builds the root Model rule from whatever imports are actually resolved - language-model-csv now builds a real contribution plugin (external references for Class/ID/STRING/NEWLINE so its serialized grammar doesn't duplicate definitions already provided by the host) - New service-model-csv microservice mirrors service-config-mdeo: registers the CSV contribution with the model language and hosts its standalone generated language service - Deleted the dead mutable-registry scaffolding (registerModelContributionPlugin) that caused the original ESM dual-instance bug, and the orphaned files left over from the previous abandoned attempt - modelScopeProvider's class-reference resolution is now plugin-agnostic (dispatches on the "class" property generically instead of hardcoding CsvClassImport by type) - Fixed a pre-existing bug in modelDataHandler where the CSV file path was passed as a full URI to an API expecting a plain relative path Verified end to end: grammar parses with no errors, live diagram rendering from CSV still works, and the model-data execution pipeline correctly resolves CSV-backed instances. --- app/packages/language-model-csv/package.json | 3 +- .../language-model-csv/src/csvPlugin.ts | 90 +++++ .../src/features/csvImport.ts | 2 +- .../src/grammar/csvImportRules.ts | 36 +- .../src/grammar/csvImportTypes.ts | 26 +- app/packages/language-model-csv/src/index.ts | 2 + .../src/plugin/modelCsvContributionPlugin.ts | 61 ++- app/packages/language-model-csv/tsconfig.json | 3 +- .../language-model/src/csv/csvImportRules.ts | 21 - .../language-model/src/csv/csvImportTypes.ts | 15 - .../diagram-server/modelGModelFactory.ts | 23 +- .../diagram-server/modelMetadataManager.ts | 6 +- .../src/features/modelScopeProvider.ts | 15 +- .../language-model/src/grammar/modelRules.ts | 61 +-- .../language-model/src/grammar/modelTypes.ts | 33 +- app/packages/language-model/src/index.ts | 5 +- .../src/modelContributionPlugin.ts | 17 - .../language-model/src/modelPlugin.ts | 53 ++- .../src/plugin/modelContributionPlugin.ts | 87 +++++ .../src/plugin/resolvePlugins.ts | 369 ++++++++++++++++++ app/packages/service-model-csv/package.json | 30 ++ app/packages/service-model-csv/src/index.ts | 78 ++++ .../service-model-csv/src/served/language.ts | 7 + app/packages/service-model-csv/tsconfig.json | 17 + app/packages/service-model-csv/vite.config.ts | 16 + .../src/handler/modelDataHandler.ts | 18 +- app/packages/service-model/src/index.ts | 5 +- app/tsconfig.build.json | 6 + infra/docker-compose-dev.yaml | 26 +- infra/docker/service-model-csv/Dockerfile | 38 ++ infra/docker/workbench/nginx.conf | 14 + 31 files changed, 991 insertions(+), 192 deletions(-) create mode 100644 app/packages/language-model-csv/src/csvPlugin.ts rename app/packages/{language-model => language-model-csv}/src/features/csvImport.ts (99%) delete mode 100644 app/packages/language-model/src/csv/csvImportRules.ts delete mode 100644 app/packages/language-model/src/csv/csvImportTypes.ts delete mode 100644 app/packages/language-model/src/modelContributionPlugin.ts create mode 100644 app/packages/language-model/src/plugin/modelContributionPlugin.ts create mode 100644 app/packages/language-model/src/plugin/resolvePlugins.ts create mode 100644 app/packages/service-model-csv/package.json create mode 100644 app/packages/service-model-csv/src/index.ts create mode 100644 app/packages/service-model-csv/src/served/language.ts create mode 100644 app/packages/service-model-csv/tsconfig.json create mode 100644 app/packages/service-model-csv/vite.config.ts create mode 100644 infra/docker/service-model-csv/Dockerfile diff --git a/app/packages/language-model-csv/package.json b/app/packages/language-model-csv/package.json index 56c1fd12..40044e22 100644 --- a/app/packages/language-model-csv/package.json +++ b/app/packages/language-model-csv/package.json @@ -8,7 +8,8 @@ "dependencies": { "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", - "@mdeo/language-metamodel": "^0.1.0" + "@mdeo/language-metamodel": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" }, "devDependencies": { "@types/node": "^25.3.1", diff --git a/app/packages/language-model-csv/src/csvPlugin.ts b/app/packages/language-model-csv/src/csvPlugin.ts new file mode 100644 index 00000000..541d0d5d --- /dev/null +++ b/app/packages/language-model-csv/src/csvPlugin.ts @@ -0,0 +1,90 @@ +import { + type LangiumLanguagePlugin, + type LangiumLanguagePluginProvider, + type ExternalReferenceAdditionalServices, + GrammarDeserializationContext, + ID, + NEWLINE, + HIDDEN_NEWLINE, + INT, + FLOAT, + STRING +} from "@mdeo/language-common"; +import { + DefaultAstSerializer, + IdValueConverter, + SerializerFormatter, + registerDefaultTokenSerializers, + NewlineAwareTokenBuilder, + addExternalReferenceCollectionPhase, + ActionHandlerRegistry, + type ActionHandlerRegistryAdditionalServices, + DefaultActionProvider +} from "@mdeo/language-shared"; +import { generateModelContributionGrammar, ModelTerminals, ModelScopeProvider, ModelExternalReferenceCollector } from "@mdeo/language-model"; +import { Class } from "@mdeo/language-metamodel"; +import { createModelCsvContributionPlugin } from "./plugin/modelCsvContributionPlugin.js"; + +/** + * Combined services type for the standalone model-csv language. + * Reuses the base Model language's services, since the standalone grammar's + * root is a real `Model` node (an `import` line plus the CSV import block). + */ +export type ModelCsvServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; + +/** + * Deserialization context for the CSV grammar. + * Provides the external interface types and common terminals so the CSV + * grammar can be deserialized from its serialized form. + */ +const csvDeserializationContext = GrammarDeserializationContext.create([Class], [], [ID, NEWLINE, HIDDEN_NEWLINE, INT, FLOAT, STRING]); + +/** + * The root rule for the standalone model-csv language. + * Generated from the CSV contribution plugin's grammar. + */ +const csvRootRule = generateModelContributionGrammar(createModelCsvContributionPlugin(), csvDeserializationContext); + +/** + * The plugin for the standalone model-csv generated language. + * Its root type is a real `Model` node, so it reuses the base Model language's + * own scope provider and external reference collector directly. + */ +const modelCsvPlugin: LangiumLanguagePlugin = { + rootRule: csvRootRule, + additionalTerminals: ModelTerminals, + module: { + parser: { + TokenBuilder: () => new NewlineAwareTokenBuilder(new Set(["{"]), new Set(["("]), new Set(["}", ")"])), + ValueConverter: () => new IdValueConverter(), + ParserConfig: () => ({ + maxLookahead: 4 + }) + }, + references: { + ScopeProvider: (services) => new ModelScopeProvider(services), + ExternalReferenceCollector: () => new ModelExternalReferenceCollector() + }, + lsp: { + Formatter: (services) => new SerializerFormatter(services) + }, + AstSerializer: (services) => new DefaultAstSerializer(services), + action: { + ActionHandlerRegistry: () => new ActionHandlerRegistry(), + ActionProvider: () => new DefaultActionProvider() + } + }, + postCreate(services) { + registerDefaultTokenSerializers(services); + addExternalReferenceCollectionPhase(services); + } +}; + +/** + * Provider for the standalone model-csv language plugin. + */ +export const modelCsvPluginProvider: LangiumLanguagePluginProvider = { + create(): LangiumLanguagePlugin { + return modelCsvPlugin; + } +}; diff --git a/app/packages/language-model/src/features/csvImport.ts b/app/packages/language-model-csv/src/features/csvImport.ts similarity index 99% rename from app/packages/language-model/src/features/csvImport.ts rename to app/packages/language-model-csv/src/features/csvImport.ts index 12ec6785..c72c9c08 100644 --- a/app/packages/language-model/src/features/csvImport.ts +++ b/app/packages/language-model-csv/src/features/csvImport.ts @@ -1,4 +1,4 @@ -import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "./modelData.js"; +import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "@mdeo/language-model"; export interface CsvImportEntry { className: string; diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts index 5300dc7b..af3ba60e 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportRules.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -1,21 +1,25 @@ -import { createRule, many, STRING, NEWLINE, ID, or, ref } from "@mdeo/language-common"; -import { Class } from "@mdeo/language-metamodel"; -import { CsvClassImport, CsvImportBlock } from "./csvImportTypes.js"; +import { createRule, many, or, ref, createExternalTerminalRule } from "@mdeo/language-common"; +import { CsvClassImport, CsvImportBlock, ExternalClass } from "./csvImportTypes.js"; + +/** + * Stand-ins for the base language's common terminals, so this grammar's + * serialized form marks them as external rather than inlining duplicate + * definitions. The real terminals are supplied via the deserialization + * context wherever this grammar is merged in. + */ +const ID = createExternalTerminalRule("ID"); +const STRING = createExternalTerminalRule("STRING"); +const NEWLINE = createExternalTerminalRule("NEWLINE"); export const CsvClassImportRule = createRule("CsvClassImportRule") .returns(CsvClassImport) - .as(({ set }) => [ - set("className", ref(Class, ID)), - "from", - set("file", STRING) - ]); + .as(({ set }) => [set("class", ref(ExternalClass, ID)), "from", set("file", STRING)]); -export const CsvImportBlockRule = createRule("CsvImportBlockRule") +/** + * The content of a CSV import block (everything between the braces). + * The `import CSV` keywords themselves are added by the wrapper rule the + * Model language builds around this contribution, not by this rule. + */ +export const CsvImportContentRule = createRule("CsvImportContentRule") .returns(CsvImportBlock) - .as(({ add }) => [ - "import", - "CSV", - "{", - many(or(add("imports", CsvClassImportRule), NEWLINE)), - "}" - ]); + .as(({ add }) => ["{", many(or(add("imports", CsvClassImportRule), NEWLINE)), "}"]); diff --git a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts index cb4fac64..e55ce295 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts @@ -1,13 +1,33 @@ -import { createInterface, Optional, Ref, type ASTType } from "@mdeo/language-common"; -import { Class } from "@mdeo/language-metamodel"; +import { createInterface, createExternalInterface, Ref, type ASTType } from "@mdeo/language-common"; +import type { ClassType } from "@mdeo/language-metamodel"; +/** + * Stand-in for the metamodel's Class interface, used only so this grammar's + * serialized form marks cross-references to it as external rather than + * inlining a duplicate "Class" interface definition. The real Class type is + * supplied via the deserialization context wherever this grammar is merged in. + */ +export const ExternalClass = createExternalInterface("Class"); + +/** + * A single CSV class import entry. + * Maps a metamodel class to a CSV file. + * Format: ClassName from "path/to/file.csv" + */ export const CsvClassImport = createInterface("CsvClassImport").attrs({ - className: Ref(() => Class), + class: Ref(() => ExternalClass), file: String }); export type CsvClassImportType = ASTType; +/** + * The content of a CSV import block (everything between the braces). + * Format: + * { + * ClassName from "file.csv" + * } + */ export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ imports: [CsvClassImport] }); diff --git a/app/packages/language-model-csv/src/index.ts b/app/packages/language-model-csv/src/index.ts index cd29dbd9..a79e548c 100644 --- a/app/packages/language-model-csv/src/index.ts +++ b/app/packages/language-model-csv/src/index.ts @@ -1,3 +1,5 @@ export * from "./grammar/csvImportTypes.js"; export * from "./grammar/csvImportRules.js"; export * from "./plugin/modelCsvContributionPlugin.js"; +export * from "./features/csvImport.js"; +export * from "./csvPlugin.js"; diff --git a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts index df6e15d4..16704c3b 100644 --- a/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts +++ b/app/packages/language-model-csv/src/plugin/modelCsvContributionPlugin.ts @@ -1,16 +1,57 @@ -import { type ModelContributionPlugin, MODEL_CONTRIBUTION_PLUGIN_TYPE } from "@mdeo/language-model"; -import { CsvClassImportRule, CsvImportBlockRule } from "../grammar/csvImportRules.js"; +import { ModelContributionPlugin } from "@mdeo/language-model"; +import { GrammarSerializer, type SerializedGrammar } from "@mdeo/language-common"; +import { CsvClassImportRule, CsvImportContentRule } from "../grammar/csvImportRules.js"; +import { CsvImportBlock } from "../grammar/csvImportTypes.js"; -export const MODEL_CSV_CONTRIBUTION_PLUGIN_ID = "model-csv"; +/** + * The unique name for the CSV contribution plugin. + */ +export const CSV_PLUGIN_NAME = "csv"; +/** + * The unique plugin ID for the CSV contribution plugin. + */ +export const CSV_PLUGIN_ID = "model-csv"; + +/** + * The language key used to retrieve services for the standalone model-csv language. + */ +export const MODEL_CSV_LANGUAGE_KEY = "model-csv"; + +/** + * Creates the serialized grammar for the CSV import rules. + * + * @returns The serialized grammar + */ +function createCsvGrammar(): SerializedGrammar { + const serializer = new GrammarSerializer({ + rules: [CsvClassImportRule, CsvImportContentRule], + additionalTerminals: [] + }); + return serializer.grammar; +} + +/** + * Creates the CSV contribution plugin. + * This plugin provides the `import CSV { ClassName from "file.csv" }` syntax for the model language. + * + * @returns The ModelContributionPlugin for CSV + */ export function createModelCsvContributionPlugin(): ModelContributionPlugin { return { - type: MODEL_CONTRIBUTION_PLUGIN_TYPE, - id: MODEL_CSV_CONTRIBUTION_PLUGIN_ID, - name: "CSV Import", - additionalTerminals: [], - additionalRules: [CsvClassImportRule, CsvImportBlockRule], - topLevelRuleNames: [CsvImportBlockRule.name], - keywords: ["import", "CSV", "from"] + id: CSV_PLUGIN_ID, + type: ModelContributionPlugin.TYPE, + name: CSV_PLUGIN_NAME, + languageKey: MODEL_CSV_LANGUAGE_KEY, + grammar: createCsvGrammar(), + imports: [ + { + name: "CSV", + ruleName: CsvImportContentRule.name, + interfaceName: CsvImportBlock.name + } + ], + dependencies: [], + exportedTypes: [] }; } diff --git a/app/packages/language-model-csv/tsconfig.json b/app/packages/language-model-csv/tsconfig.json index 224fb6ed..a2d1512f 100644 --- a/app/packages/language-model-csv/tsconfig.json +++ b/app/packages/language-model-csv/tsconfig.json @@ -9,6 +9,7 @@ "references": [ { "path": "../language-common" }, { "path": "../language-model" }, - { "path": "../language-metamodel" } + { "path": "../language-metamodel" }, + { "path": "../language-shared" } ] } diff --git a/app/packages/language-model/src/csv/csvImportRules.ts b/app/packages/language-model/src/csv/csvImportRules.ts deleted file mode 100644 index 5300dc7b..00000000 --- a/app/packages/language-model/src/csv/csvImportRules.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createRule, many, STRING, NEWLINE, ID, or, ref } from "@mdeo/language-common"; -import { Class } from "@mdeo/language-metamodel"; -import { CsvClassImport, CsvImportBlock } from "./csvImportTypes.js"; - -export const CsvClassImportRule = createRule("CsvClassImportRule") - .returns(CsvClassImport) - .as(({ set }) => [ - set("className", ref(Class, ID)), - "from", - set("file", STRING) - ]); - -export const CsvImportBlockRule = createRule("CsvImportBlockRule") - .returns(CsvImportBlock) - .as(({ add }) => [ - "import", - "CSV", - "{", - many(or(add("imports", CsvClassImportRule), NEWLINE)), - "}" - ]); diff --git a/app/packages/language-model/src/csv/csvImportTypes.ts b/app/packages/language-model/src/csv/csvImportTypes.ts deleted file mode 100644 index cb4fac64..00000000 --- a/app/packages/language-model/src/csv/csvImportTypes.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { createInterface, Optional, Ref, type ASTType } from "@mdeo/language-common"; -import { Class } from "@mdeo/language-metamodel"; - -export const CsvClassImport = createInterface("CsvClassImport").attrs({ - className: Ref(() => Class), - file: String -}); - -export type CsvClassImportType = ASTType; - -export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ - imports: [CsvClassImport] -}); - -export type CsvImportBlockType = ASTType; diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index f7fd07f8..65df80f0 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -34,6 +34,7 @@ import { type EnumValueType, type SingleValueType } from "../../grammar/modelTypes.js"; +import { getWrapperInterfaceName } from "../../plugin/resolvePlugins.js"; const { injectable } = sharedImport("inversify"); const { GGraph } = sharedImport("@eclipse-glsp/server"); @@ -463,15 +464,33 @@ export class ModelGModelFactory extends BaseGModelFactory { return nodes; } + /** + * Renders diagram nodes for the CSV import contribution, if present. + * + * The Model language's grammar has no compile-time knowledge of CSV (it's a + * plugin contribution resolved generically via `model.imports`), but rendering + * synthetic instance nodes from CSV row data has no equivalent generic extension + * point today, so this reads the CSV plugin's contributed import structurally + * by its wrapper type name rather than importing its types directly (which would + * create a circular package dependency, since language-model-csv depends on + * language-model for the contribution plugin contract). + */ private async createCsvNodes(graph: GGraphType, model: PartialModel): Promise { - if (model.csvImport == undefined) return; + interface CsvClassImportShape { + class?: { ref?: ClassType }; + file?: string; + } + + const csvImport = model.imports?.find((imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV")); + const csvImportContent = (csvImport as { content?: { imports?: CsvClassImportShape[] } } | undefined)?.content; + if (csvImportContent?.imports == undefined) return; const doc = this.modelState.sourceModel?.$document; if (!doc) return; const validatedMetadata = await this.modelState.getValidatedMetadata(); let nodeIndex = 0; - for (const entry of model.csvImport.imports ?? []) { + for (const entry of csvImportContent.imports) { const classRef = entry.class?.ref as ClassType | undefined; if (classRef == undefined) continue; try { diff --git a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts index b5bf73c2..13847e6d 100644 --- a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts +++ b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts @@ -13,6 +13,7 @@ import { } from "@mdeo/language-shared"; import { ModelElementType } from "@mdeo/protocol-model"; import type { PartialModel, PartialObjectInstance, PartialLink } from "../../grammar/modelPartialTypes.js"; +import { getWrapperInterfaceName } from "../../plugin/resolvePlugins.js"; const { injectable, inject } = sharedImport("inversify"); @@ -80,7 +81,10 @@ export class ModelMetadataManager extends MetadataManager { } private extractCsvNodeMetadata(sourceModel: PartialModel, nodes: Record): void { - if (sourceModel.csvImport == undefined) return; + const hasCsvImport = sourceModel.imports?.some( + (imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV") + ); + if (!hasCsvImport) return; if (this.storedCurrentMetadata == undefined) return; for (const [id, meta] of Object.entries(this.storedCurrentMetadata.nodes)) { if (id.startsWith("csv-node-") && !nodes[id]) { diff --git a/app/packages/language-model/src/features/modelScopeProvider.ts b/app/packages/language-model/src/features/modelScopeProvider.ts index e9754963..5ff5af84 100644 --- a/app/packages/language-model/src/features/modelScopeProvider.ts +++ b/app/packages/language-model/src/features/modelScopeProvider.ts @@ -3,13 +3,11 @@ import { sharedImport, resolveRelativePath } from "@mdeo/language-shared"; import type { AstNodeDescriptionProvider, LangiumDocuments, ReferenceInfo, Scope } from "langium"; import { ObjectInstance, - CsvClassImport, PropertyAssignment, EnumValue, LinkEnd, type ModelType, type ObjectInstanceType, - type CsvClassImportType, type PropertyAssignmentType, type LinkEndType, type EnumValueType, @@ -74,7 +72,7 @@ export class ModelScopeProvider extends DefaultScopeProvider { override getScope(context: ReferenceInfo): Scope { const document = AstUtils.getDocument(context.container); - if (context.property === "class" && (this.astReflection.isInstance(context.container, ObjectInstance) || this.astReflection.isInstance(context.container, CsvClassImport))) { + if (context.property === "class") { return this.getObjectClassScope(context, document); } if (context.property === "name" && this.astReflection.isInstance(context.container, PropertyAssignment)) { @@ -97,16 +95,19 @@ export class ModelScopeProvider extends DefaultScopeProvider { } /** - * Gets the scope for object class references. - * Resolves the imported metamodel file and returns a scope with all accessible classes. + * Gets the scope for class references, wherever they occur in the document + * (an object instance, or a class reference inside a plugin-contributed import + * such as CSV, at any nesting depth). Resolves the imported metamodel file and + * returns a scope with all accessible classes. * * @param context The reference context * @param document The current document * @returns A scope containing all accessible classes from the imported metamodel */ private getObjectClassScope(context: ReferenceInfo, document: any): Scope { - const parent = context.container.$container; - const model = (this.astReflection.isInstance(parent, Model) ? parent : parent?.$container) as ModelType; + const model = AstUtils.getContainerOfType(context.container, (node): node is ModelType => + this.astReflection.isInstance(node, Model) + ); const metamodelImport = model?.import; const relativePath = metamodelImport?.file; diff --git a/app/packages/language-model/src/grammar/modelRules.ts b/app/packages/language-model/src/grammar/modelRules.ts index ab39522f..6f06a9a2 100644 --- a/app/packages/language-model/src/grammar/modelRules.ts +++ b/app/packages/language-model/src/grammar/modelRules.ts @@ -14,7 +14,8 @@ import { ML_COMMENT, SL_COMMENT, HIDDEN_NEWLINE, - type ParserRule + type ParserRule, + type RuleEntry } from "@mdeo/language-common"; import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import { @@ -28,9 +29,7 @@ import { Model, MetamodelFileImport, LinkEnd, - Link, - CsvClassImport, - CsvImportBlock + Link } from "./modelTypes.js"; export const BOOLEAN = createRule("BOOLEAN") @@ -93,40 +92,42 @@ export const MetamodelFileImportRule = createRule("MetamodelFileImportRule") .returns(MetamodelFileImport) .as(({ set }) => ["using", set("file", STRING)]); -export const CsvClassImportRule = createRule("CsvClassImportRule") - .returns(CsvClassImport) - .as(({ set }) => [ - set("class", ref(Class, ID)), - "from", - set("file", STRING) - ]); - -export const CsvImportBlockRule = createRule("CsvImportBlockRule") - .returns(CsvImportBlock) - .as(({ add }) => [ - "import", - "CSV", - "{", - many(or(add("imports", CsvClassImportRule), NEWLINE)), - "}" - ]); - +/** + * Default root rule for the Model language (without contributed imports). + * Use createModelRule() with resolved plugins for the full grammar. + */ export const ModelRule = createRule("ModelRule") .returns(Model) .as(({ add, set }) => [ many(NEWLINE), set("import", MetamodelFileImportRule), many(NEWLINE), - many(or( - set("csvImport", CsvImportBlockRule), - add("objects", ObjectInstanceRule), - add("links", LinkRule), - NEWLINE - )) + many(or(add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) ]); -export function createModelRule(_contributedTopLevelRules: ParserRule[] = []): ParserRule { - return ModelRule; +/** + * Builds the root Model rule, alternating between hand-authored objects/links + * and any data imports contributed by plugins (e.g. CSV). + * + * @param contributedImportRules Wrapper rules for imports contributed by resolved plugins + * @returns The root parser rule for the Model language + */ +export function createModelRule(contributedImportRules: ParserRule[] = []): ParserRule { + if (contributedImportRules.length === 0) { + return ModelRule; + } + + return createRule("ModelRule") + .returns(Model) + .as(({ add, set }) => { + const importAlternatives: RuleEntry[] = contributedImportRules.map((rule) => add("imports", rule)); + return [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or(...importAlternatives, add("objects", ObjectInstanceRule), add("links", LinkRule), NEWLINE)) + ]; + }); } export const ModelTerminals = [WS, HIDDEN_NEWLINE, ML_COMMENT, SL_COMMENT]; diff --git a/app/packages/language-model/src/grammar/modelTypes.ts b/app/packages/language-model/src/grammar/modelTypes.ts index 70083c10..40e79ca0 100644 --- a/app/packages/language-model/src/grammar/modelTypes.ts +++ b/app/packages/language-model/src/grammar/modelTypes.ts @@ -67,39 +67,22 @@ export const MetamodelFileImport = createInterface("MetamodelFileImport").attrs( export type MetamodelFileImportType = ASTType; /** - * A single CSV class import entry. - * Maps a metamodel class to a CSV file. - * Format: ClassName from "path/to/file.csv" + * Base interface for data imports contributed by plugins (e.g. CSV, and future formats). + * A contribution plugin wraps its own import block in an interface extending this one, + * the same way config sections extend `BaseConfigSection`. */ -export const CsvClassImport = createInterface("CsvClassImport").attrs({ - class: Ref(() => Class), - file: String -}); - -export type CsvClassImportType = ASTType; - -/** - * A CSV import block at the top of a model file. - * When present, forbids hand-authored object instances. - * Format: - * import CSV { - * ClassName from "file.csv" - * } - */ -export const CsvImportBlock = createInterface("CsvImportBlock").attrs({ - imports: [CsvClassImport] -}); +export const BaseModelImport = createInterface("BaseModelImport").attrs({}); -export type CsvImportBlockType = ASTType; +export type BaseModelImportType = ASTType; /** * Model root interface. - * Contains metamodel import, optional CSV import block, optional hand-authored objects and links. - * Either csvImport or objects/links may be present, but not both. + * Contains metamodel import, any number of plugin-contributed data imports (e.g. CSV), + * and hand-authored objects and links. */ export const Model = createInterface("Model").attrs({ import: MetamodelFileImport, - csvImport: Optional(CsvImportBlock), + imports: [BaseModelImport], objects: [ObjectInstance], links: [Link] }); diff --git a/app/packages/language-model/src/index.ts b/app/packages/language-model/src/index.ts index 6025cae3..d065aa7f 100644 --- a/app/packages/language-model/src/index.ts +++ b/app/packages/language-model/src/index.ts @@ -1,5 +1,4 @@ export * from "./action-handlers/newFileActionHandler.js"; -export * from "./features/csvImport.js"; export * from "./action-handlers/saveGeneratedModelActionHandler.js"; export * from "./features/diagram-server/generated/generatedModelCreateEdgeSchemaResolver.js"; export * from "./features/diagram-server/generated/generatedModelDiagramConfiguration.js"; @@ -54,5 +53,5 @@ export * from "./validation/baseModelValidator.js"; export * from "./validation/modelValidator.js"; export * from "./generatedModelPlugin.js"; export * from "./modelPlugin.js"; -export * from "./modelContributionPlugin.js"; -export { registerModelContributionPlugin } from "./modelPlugin.js"; +export * from "./plugin/modelContributionPlugin.js"; +export * from "./plugin/resolvePlugins.js"; diff --git a/app/packages/language-model/src/modelContributionPlugin.ts b/app/packages/language-model/src/modelContributionPlugin.ts deleted file mode 100644 index 9826a1d1..00000000 --- a/app/packages/language-model/src/modelContributionPlugin.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { TerminalRule, ParserRule } from "@mdeo/language-common"; - -export const MODEL_CONTRIBUTION_PLUGIN_TYPE = "model-contribution-plugin"; - -export interface ModelContributionPlugin { - readonly type: typeof MODEL_CONTRIBUTION_PLUGIN_TYPE; - readonly id: string; - readonly name: string; - readonly additionalTerminals: TerminalRule[]; - readonly additionalRules: ParserRule[]; - readonly topLevelRuleNames: string[]; - readonly keywords: string[]; -} - -export namespace ModelContributionPlugin { - export const TYPE = MODEL_CONTRIBUTION_PLUGIN_TYPE; -} diff --git a/app/packages/language-model/src/modelPlugin.ts b/app/packages/language-model/src/modelPlugin.ts index d3974fee..5d8f2529 100644 --- a/app/packages/language-model/src/modelPlugin.ts +++ b/app/packages/language-model/src/modelPlugin.ts @@ -1,8 +1,16 @@ import { type ExternalReferenceAdditionalServices, type LangiumLanguagePlugin, - type LangiumLanguagePluginProvider + type LangiumLanguagePluginProvider, + GrammarDeserializationContext, + ID, + NEWLINE, + HIDDEN_NEWLINE, + INT, + FLOAT, + STRING } from "@mdeo/language-common"; +import { Class } from "@mdeo/language-metamodel"; import { createModelRule, ModelTerminals } from "./grammar/modelRules.js"; import { addExternalReferenceCollectionPhase, @@ -23,42 +31,31 @@ import { registerModelSerializers } from "./features/modelSerializers.js"; import { ModelDiagramModule } from "./features/diagram-server/modelDiagramModule.js"; import { registerModelValidationChecks } from "./validation/modelValidator.js"; import { ModelCompletionProvider } from "./features/modelCompletionProvider.js"; -import type { ModelContributionPlugin } from "./modelContributionPlugin.js"; +import { ModelContributionPlugin } from "./plugin/modelContributionPlugin.js"; +import { resolveModelPlugins } from "./plugin/resolvePlugins.js"; import type { ServerContributionPlugin } from "@mdeo/plugin"; -import { CsvClassImportRule, CsvImportBlockRule } from "./csv/csvImportRules.js"; export type ModelServices = ExternalReferenceAdditionalServices & ActionHandlerRegistryAdditionalServices; -const CSV_CONTRIBUTION: ModelContributionPlugin = { - type: "model-contribution-plugin", - id: "model-csv", - name: "CSV Import", - additionalTerminals: [], - additionalRules: [CsvClassImportRule, CsvImportBlockRule], - topLevelRuleNames: ["CsvImportBlockRule"], - keywords: ["import", "CSV", "from"] -}; - -function isModelContributionPlugin(plugin: ServerContributionPlugin): plugin is ModelContributionPlugin { - return (plugin as ModelContributionPlugin).topLevelRuleNames !== undefined; -} +function createModelPlugin( + contributionPlugins: ServerContributionPlugin[], + languageJsUrl?: string +): LangiumLanguagePlugin { + const modelPlugins = contributionPlugins.filter(ModelContributionPlugin.is); -function createModelPlugin(contributionPlugins: ServerContributionPlugin[], languageJsUrl?: string): LangiumLanguagePlugin { - const allContributions = [ - CSV_CONTRIBUTION, - ...contributionPlugins.filter(isModelContributionPlugin) - ]; - - const contributedTopLevelRules = allContributions.flatMap(p => - p.additionalRules.filter(r => p.topLevelRuleNames.includes(r.name)) + const deserializationContext = GrammarDeserializationContext.create( + [Class], + [], + [ID, NEWLINE, HIDDEN_NEWLINE, INT, FLOAT, STRING] ); - const contributedTerminals = allContributions.flatMap(p => p.additionalTerminals); - const ModelRule = createModelRule(contributedTopLevelRules); + const resolvedPlugins = resolveModelPlugins(modelPlugins, deserializationContext); + + const ModelRule = createModelRule(resolvedPlugins.rules); return { rootRule: ModelRule, - additionalTerminals: [...ModelTerminals, ...contributedTerminals], + additionalTerminals: ModelTerminals, module: { parser: { TokenBuilder: () => new NewlineAwareTokenBuilder( @@ -107,5 +104,3 @@ export const modelPluginProvider: LangiumLanguagePluginProvider = return createModelPlugin(contributionPlugins, languageJsUrl); } }; - -export function registerModelContributionPlugin(_plugin: ModelContributionPlugin): void {} diff --git a/app/packages/language-model/src/plugin/modelContributionPlugin.ts b/app/packages/language-model/src/plugin/modelContributionPlugin.ts new file mode 100644 index 00000000..d3bf2880 --- /dev/null +++ b/app/packages/language-model/src/plugin/modelContributionPlugin.ts @@ -0,0 +1,87 @@ +import type { ServerContributionPlugin } from "@mdeo/plugin"; +import type { SerializedGrammar } from "@mdeo/language-common"; + +/** + * A data import format that can be contributed by a plugin (e.g. CSV). + * Each plugin contributes exactly one import keyword and the rule/interface + * in its own serialized grammar that defines the content following that keyword. + */ +export interface ModelImportContribution { + /** + * The keyword introducing this import in the model file syntax (e.g. "CSV"), + * used as `import { ... }`. + */ + name: string; + + /** + * The name of the parser rule in the plugin's serialized grammar that defines + * the content of this import block (everything after the `{`). + */ + ruleName: string; + + /** + * The name of the interface in the plugin's serialized grammar that defines + * this import's AST type. + */ + interfaceName: string; +} + +/** + * Plugin for contributing a data import format to the Model language. + */ +export interface ModelContributionPlugin extends ServerContributionPlugin { + /** + * Identifies the plugin as a Model language contribution. + */ + type: typeof ModelContributionPlugin.TYPE; + + /** + * The short name of the plugin (e.g. "csv"). + */ + name: string; + + /** + * The language key used to get language services for this plugin. + * This is the language ID registered with Langium's ServiceRegistry. + */ + languageKey: string; + + /** + * The serialized grammar containing all rules for this plugin's import. + */ + grammar: SerializedGrammar; + + /** + * The import format contributed by this plugin. + */ + imports: ModelImportContribution[]; + + /** + * Plugin IDs that this plugin depends on. + * These plugins must be loaded before this one, and their exported types + * will be available in this plugin's deserialization context. + */ + dependencies: string[]; + + /** + * Names of types/interfaces exported by this plugin for use by other plugins. + */ + exportedTypes: string[]; +} + +export namespace ModelContributionPlugin { + /** + * The type identifier for ModelContributionPlugin. + */ + export const TYPE = "model-language-contribution"; + + /** + * Type guard for ModelContributionPlugin. + * + * @param value The value to check + * @returns True if the value is a ModelContributionPlugin, false otherwise + */ + export function is(value: ServerContributionPlugin): value is ModelContributionPlugin { + return "type" in value && value.type === TYPE; + } +} diff --git a/app/packages/language-model/src/plugin/resolvePlugins.ts b/app/packages/language-model/src/plugin/resolvePlugins.ts new file mode 100644 index 00000000..04315c8b --- /dev/null +++ b/app/packages/language-model/src/plugin/resolvePlugins.ts @@ -0,0 +1,369 @@ +import { + GrammarDeserializer, + isTerminalRule, + GrammarDeserializationContext, + type Interface, + type ParserRule, + type Type, + type TerminalRule, + createInterface, + createRule, + or, + many, + NEWLINE +} from "@mdeo/language-common"; +import type { ModelContributionPlugin, ModelImportContribution } from "./modelContributionPlugin.js"; +import { BaseModelImport, Model } from "../grammar/modelTypes.js"; +import { MetamodelFileImportRule } from "../grammar/modelRules.js"; +import type { AstNode } from "langium"; + +/** + * Naming and grammar information for a resolved import contribution. + */ +export interface ImportNamingInfo { + /** + * The keyword introducing this import (e.g. "CSV"). + */ + importName: string; + + /** + * The plugin that contributes this import. + */ + plugin: ModelContributionPlugin; + + /** + * The resolved wrapper interface for this import, extending BaseModelImport. + */ + interface: Interface; + + /** + * The resolved wrapper parser rule for this import. + */ + rule: ParserRule; +} + +/** + * Resolved plugins containing the contributions of all plugins. + */ +export interface ResolvedModelContributionPlugins { + /** + * All import naming info, indexed by the import keyword. + */ + imports: Map; + + /** + * Keywords introduced by resolved plugins. + */ + keywords: string[]; + + /** + * All wrapper parser rules from all plugins. + */ + rules: ParserRule[]; +} + +/** + * Sorts plugins by dependency order using topological sort. + * + * @param plugins The contribution plugins to sort + * @returns The sorted plugins + * @throws Error if circular dependencies are detected + */ +function sortPluginsByDependencies(plugins: ModelContributionPlugin[]): ModelContributionPlugin[] { + const sorted: ModelContributionPlugin[] = []; + const visited = new Set(); + const visiting = new Set(); + const pluginMap = new Map(); + + for (const plugin of plugins) { + pluginMap.set(plugin.id, plugin); + } + + function visit(pluginId: string): void { + if (visited.has(pluginId)) { + return; + } + if (visiting.has(pluginId)) { + throw new Error(`Circular dependency detected involving plugin: ${pluginId}`); + } + + visiting.add(pluginId); + + const plugin = pluginMap.get(pluginId); + if (plugin) { + for (const depId of plugin.dependencies) { + const depPlugin = pluginMap.get(depId); + if (!depPlugin) { + throw new Error(`Plugin '${pluginId}' depends on '${depId}' which is not available`); + } + visit(depId); + } + visited.add(pluginId); + visiting.delete(pluginId); + sorted.push(plugin); + } + } + + for (const plugin of plugins) { + visit(plugin.id); + } + + return sorted; +} + +/** + * Class responsible for resolving model contribution plugins. + * Encapsulates the state and logic for processing plugins and their imports. + */ +class ModelPluginResolver { + private readonly imports: Map; + private readonly keywords: string[]; + private readonly rules: ParserRule[]; + private readonly exportedTypes: (Interface | Type)[]; + private readonly baseTypes: (Interface | Type)[]; + private readonly baseRules: ParserRule[]; + private readonly baseTerminals: TerminalRule[]; + + constructor(deserializationContext: GrammarDeserializationContext) { + this.imports = new Map(); + this.keywords = []; + this.rules = []; + this.exportedTypes = []; + this.baseTypes = Array.from(deserializationContext.types?.values() ?? []); + this.baseRules = Array.from(deserializationContext.parserRules?.values() ?? []); + this.baseTerminals = Array.from(deserializationContext.terminalRules?.values() ?? []); + } + + /** + * Resolves all plugins and returns the combined result. + * + * @param sortedPlugins The plugins to process, sorted by dependency order + * @returns The resolved plugins with naming information + */ + resolve(sortedPlugins: ModelContributionPlugin[]): ResolvedModelContributionPlugins { + for (const plugin of sortedPlugins) { + this.processPlugin(plugin); + } + + return { + imports: this.imports, + keywords: this.keywords, + rules: this.rules + }; + } + + /** + * Processes a single plugin, deserializing its grammar and processing its import contribution. + * + * @param plugin The plugin to process + */ + private processPlugin(plugin: ModelContributionPlugin): void { + const pluginDeserializationContext = GrammarDeserializationContext.create( + [...this.baseTypes, ...this.exportedTypes], + this.baseRules, + this.baseTerminals + ); + + const deserializer = new GrammarDeserializer(plugin.grammar, pluginDeserializationContext); + const grammar = deserializer.deserializeGrammar(); + + const ruleMap = new Map>(); + for (const rule of grammar.rules) { + if (!isTerminalRule(rule)) { + ruleMap.set(rule.name, rule); + } + } + + const interfaceMap = new Map>(); + for (const iface of grammar.interfaces) { + interfaceMap.set(iface.name, iface); + } + + const typeMap = new Map>(); + for (const type of grammar.types) { + typeMap.set(type.name, type); + } + + for (const imp of plugin.imports) { + this.processImport(imp, plugin, ruleMap, interfaceMap); + } + + for (const exportedTypeName of plugin.exportedTypes) { + const exportedType = interfaceMap.get(exportedTypeName) ?? typeMap.get(exportedTypeName); + if (!exportedType) { + throw new Error( + `Plugin '${plugin.id}' exports type '${exportedTypeName}' which does not exist in its grammar.` + ); + } + this.exportedTypes.push(exportedType); + } + } + + /** + * Processes a single import contribution from a plugin. + * Creates the wrapper interface and rule, and registers the import's keyword. + * + * @param imp The import contribution to process + * @param plugin The plugin contributing the import + * @param ruleMap Map of rule names to rules in the plugin's grammar + * @param interfaceMap Map of interface names to interfaces in the plugin's grammar + */ + private processImport( + imp: ModelImportContribution, + plugin: ModelContributionPlugin, + ruleMap: Map>, + interfaceMap: Map> + ): void { + if (this.imports.has(imp.name)) { + const existing = this.imports.get(imp.name)!; + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' conflicts with the same import already contributed by plugin '${existing.plugin.id}'.` + ); + } + + const importRule = ruleMap.get(imp.ruleName); + if (importRule == undefined) { + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' references rule '${imp.ruleName}' which does not exist in the plugin's grammar.` + ); + } + + const importInterface = interfaceMap.get(imp.interfaceName); + if (importInterface == undefined) { + throw new Error( + `Import '${imp.name}' from plugin '${plugin.id}' references interface '${imp.interfaceName}' which does not exist in the plugin's grammar.` + ); + } + + const { wrapperInterface, wrapperRule } = createImportWrapper(imp.name, importRule, importInterface); + + this.rules.push(wrapperRule); + this.imports.set(imp.name, { importName: imp.name, plugin, interface: wrapperInterface, rule: wrapperRule }); + this.keywords.push(imp.name); + } +} + +/** + * Creates the wrapper interface and rule for an import contribution. + * The wrapper wraps the import's content interface in a BaseModelImport, gated by its keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @param contentRule The parser rule for the import's content + * @param contentInterface The interface for the import's content + * @returns The wrapper interface and rule + */ +function createImportWrapper( + importName: string, + contentRule: ParserRule, + contentInterface: Interface +): { wrapperInterface: Interface; wrapperRule: ParserRule } { + const wrapperInterfaceName = getWrapperInterfaceName(importName); + const wrapperInterface = createInterface(wrapperInterfaceName).extends(BaseModelImport).attrs({ + content: contentInterface + }); + + const wrapperRuleName = getWrapperRuleName(importName); + const wrapperRule = createRule(wrapperRuleName) + .returns(wrapperInterface) + .as(({ set }) => ["import", importName, set("content", contentRule)]); + + return { wrapperInterface, wrapperRule }; +} + +/** + * Returns the wrapper interface name for a given import keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @returns The wrapper interface name (e.g. "ModelCsvImport") + */ +export function getWrapperInterfaceName(importName: string): string { + return `Model${capitalize(importName)}Import`; +} + +/** + * Returns the wrapper parser rule name for a given import keyword. + * + * @param importName The import keyword (e.g. "CSV") + * @returns The wrapper rule name (e.g. "ModelCsvImportWrapper") + */ +export function getWrapperRuleName(importName: string): string { + return `Model${capitalize(importName)}ImportWrapper`; +} + +function capitalize(value: string): string { + return value.length === 0 ? value : value.charAt(0).toUpperCase() + value.slice(1).toLowerCase(); +} + +/** + * Generates the grammar for the standalone (artificial) language of an import contribution plugin. + * This is used by the plugin's own language service to parse partial model files that + * contain only this plugin's import block, alongside the base "using" metamodel import. + * + * @param plugin The contribution plugin whose standalone grammar is to be generated + * @param deserializationContext Context for resolving external type/rule references in the plugin's grammar + * @returns The root parser rule for the standalone grammar + */ +export function generateModelContributionGrammar( + plugin: ModelContributionPlugin, + deserializationContext: GrammarDeserializationContext +): ParserRule { + const deserializer = new GrammarDeserializer(plugin.grammar, deserializationContext); + const grammar = deserializer.deserializeGrammar(); + + const ruleMap = new Map>(); + for (const rule of grammar.rules) { + if (!isTerminalRule(rule)) { + ruleMap.set(rule.name, rule); + } + } + + const interfaceMap = new Map>(); + for (const iface of grammar.interfaces) { + interfaceMap.set(iface.name, iface); + } + + const wrapperRules: ParserRule[] = []; + + for (const imp of plugin.imports) { + const contentRule = ruleMap.get(imp.ruleName); + if (contentRule == undefined) { + throw new Error(`Import '${imp.name}' in plugin '${plugin.id}' references rule '${imp.ruleName}' which does not exist.`); + } + + const contentInterface = interfaceMap.get(imp.interfaceName); + if (contentInterface == undefined) { + throw new Error( + `Import '${imp.name}' in plugin '${plugin.id}' references interface '${imp.interfaceName}' which does not exist.` + ); + } + + const { wrapperRule } = createImportWrapper(imp.name, contentRule, contentInterface); + wrapperRules.push(wrapperRule); + } + + return createRule(`${capitalize(plugin.name)}PluginModelRule`) + .returns(Model) + .as(({ add, set }) => [ + many(NEWLINE), + set("import", MetamodelFileImportRule), + many(NEWLINE), + many(or(...wrapperRules.map((r) => add("imports", r)), NEWLINE)) + ]); +} + +/** + * Resolves model contribution plugins in dependency order, deserializing their + * grammars and wrapping each contributed import in a BaseModelImport-extending rule. + * + * @param plugins The contribution plugins + * @param deserializationContext The deserialization context for resolving external references + * @returns The resolved plugins with naming information + */ +export function resolveModelPlugins( + plugins: ModelContributionPlugin[], + deserializationContext: GrammarDeserializationContext +): ResolvedModelContributionPlugins { + const sortedPlugins = sortPluginsByDependencies(plugins); + const resolver = new ModelPluginResolver(deserializationContext); + return resolver.resolve(sortedPlugins); +} diff --git a/app/packages/service-model-csv/package.json b/app/packages/service-model-csv/package.json new file mode 100644 index 00000000..4a707fd1 --- /dev/null +++ b/app/packages/service-model-csv/package.json @@ -0,0 +1,30 @@ +{ + "name": "@mdeo/service-model-csv", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": ["dist", "src"], + "scripts": { + "build:static": "vite build", + "watch:static": "vite build --watch", + "start": "node dist/index.js", + "watch": "tsx watch src/index.ts" + }, + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-model-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1", + "tsx": "~4.21.0", + "vite": "~7.3.1" + } +} diff --git a/app/packages/service-model-csv/src/index.ts b/app/packages/service-model-csv/src/index.ts new file mode 100644 index 00000000..f7aa15ac --- /dev/null +++ b/app/packages/service-model-csv/src/index.ts @@ -0,0 +1,78 @@ +import { Table } from "lucide"; +import { convertIcon } from "@mdeo/language-common"; +import type { ModelCsvServices } from "@mdeo/language-model-csv"; +import { + parseServiceConfigFromEnv, + type ServiceConfig, + type ServicePluginDefinition, + type LanguageServiceConfig, + initializePluginContext, + astHandler, + AST_HANDLER_KEY, + startLanguageService +} from "@mdeo/service-common"; +import type { LanguagePlugin } from "@mdeo/plugin"; + +const icon = convertIcon(Table); + +/** + * Language plugin definition for the model-csv language. + * This is a generated language that provides services for the CSV import contribution. + */ +const modelCsvLanguagePlugin: LanguagePlugin = { + id: "model-csv", + name: "Model CSV", + extension: undefined, + newFileAction: false, + icon, + serverPlugin: { + import: "language.js" + }, + graphicalEditorPlugin: undefined, + textualEditorPlugin: undefined, + isGenerated: true +}; + +initializePluginContext(); + +const { modelCsvPluginProvider, createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); + +/** + * Plugin definition for the model-csv service. + */ +const modelCsvServicePlugin: ServicePluginDefinition = { + id: "model-csv-service", + name: "Model CSV", + description: "Language support for the CSV import contribution to the model language", + icon, + languagePlugins: [modelCsvLanguagePlugin], + contributionPlugins: [ + { + languageId: "model", + description: "Provides import CSV { ClassName from \"file.csv\" } syntax for the model language", + additionalKeywords: ["import", "CSV", "from"], + serverContributionPlugins: [createModelCsvContributionPlugin()] + } + ] +}; + +const envConfig = parseServiceConfigFromEnv(); + +/** + * Language configuration for the model-csv language. + */ +const modelCsvLanguageConfig: LanguageServiceConfig = { + languagePlugin: modelCsvLanguagePlugin, + languagePluginProvider: modelCsvPluginProvider, + fileDataHandlers: { + [AST_HANDLER_KEY]: astHandler + } +}; + +const config: ServiceConfig = { + ...envConfig, + plugin: modelCsvServicePlugin, + languages: [modelCsvLanguageConfig] +}; + +await startLanguageService(config); diff --git a/app/packages/service-model-csv/src/served/language.ts b/app/packages/service-model-csv/src/served/language.ts new file mode 100644 index 00000000..f0bfae63 --- /dev/null +++ b/app/packages/service-model-csv/src/served/language.ts @@ -0,0 +1,7 @@ +/** + * Re-exports the model-csv language plugin provider for the language server. + * This file is served as an ESM module for the workbench language server to import. + */ +import { modelCsvPluginProvider } from "@mdeo/language-model-csv"; + +export default modelCsvPluginProvider; diff --git a/app/packages/service-model-csv/tsconfig.json b/app/packages/service-model-csv/tsconfig.json new file mode 100644 index 00000000..44840f38 --- /dev/null +++ b/app/packages/service-model-csv/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + { "path": "../plugin" }, + { "path": "../language-common" }, + { "path": "../language-model" }, + { "path": "../language-model-csv" }, + { "path": "../language-shared" }, + { "path": "../service-common" } + ], + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/app/packages/service-model-csv/vite.config.ts b/app/packages/service-model-csv/vite.config.ts new file mode 100644 index 00000000..1df064ba --- /dev/null +++ b/app/packages/service-model-csv/vite.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vite"; +import { resolve } from "path"; + +export default defineConfig({ + build: { + lib: { + entry: { + language: resolve(__dirname, "src/served/language.ts") + }, + formats: ["es"] + }, + outDir: "static", + emptyOutDir: true, + sourcemap: true + } +}); diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index 226a86ce..25675c8e 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -1,5 +1,10 @@ -import { type ModelData, type ModelServices, ModelDataConverter, Model } from "@mdeo/language-model"; -import { type MetamodelClassInfo, type MetamodelPropertyInfo, importCsvEntries } from "@mdeo/language-model"; +import { type ModelData, type ModelServices, ModelDataConverter, Model, getWrapperInterfaceName } from "@mdeo/language-model"; +import { + type MetamodelClassInfo, + type MetamodelPropertyInfo, + type CsvImportBlockType, + importCsvEntries +} from "@mdeo/language-model-csv"; import { hasErrors, type FileDataHandler } from "@mdeo/service-common"; import { resolveRelativePath } from "@mdeo/language-shared"; @@ -31,7 +36,10 @@ export const modelDataHandler: FileDataHandler throw new Error("Document root is not a Model"); } - if (model.csvImport == undefined) { + const csvImport = model.imports?.find((imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV")); + const csvImportContent = (csvImport as { content?: CsvImportBlockType } | undefined)?.content; + + if (csvImportContent == undefined) { const converter = new ModelDataConverter(reflection); const modelData = converter.convertModel(model); return { @@ -73,8 +81,8 @@ export const modelDataHandler: FileDataHandler })); const csvEntries = await Promise.all( - model.csvImport.imports.map(async (entry: any) => { - const csvPath: string = resolveRelativePath(document, entry.file).toString(); + csvImportContent.imports.map(async (entry: any) => { + const csvPath: string = resolveRelativePath(document, entry.file).path; const { content } = await serverApi.readFile(csvPath); return { className: entry.class?.ref?.name ?? "", diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index f72ad188..9771ccd8 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -43,11 +43,8 @@ const modelLanguagePlugin: LanguagePlugin = { initializePluginContext(); -const { modelPluginProvider, registerModelContributionPlugin } = await import("@mdeo/language-model"); +const { modelPluginProvider } = await import("@mdeo/language-model"); const { modelDataHandler, MODEL_DATA_HANDLER_KEY } = await import("./handler/modelDataHandler.js"); -const { createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); - -registerModelContributionPlugin(createModelCsvContributionPlugin()); const envConfig = parseServiceConfigFromEnv(); diff --git a/app/tsconfig.build.json b/app/tsconfig.build.json index 9e024644..84e38a6c 100644 --- a/app/tsconfig.build.json +++ b/app/tsconfig.build.json @@ -38,6 +38,9 @@ { "path": "packages/language-model" }, + { + "path": "packages/language-model-csv" + }, { "path": "packages/language-model-transformation" }, @@ -89,6 +92,9 @@ { "path": "packages/service-model" }, + { + "path": "packages/service-model-csv" + }, { "path": "packages/service-script" }, diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index fe4fc8be..109886e8 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -87,7 +87,7 @@ services: ADMIN_PASSWORD: admin PLUGIN_BASE_URL: "" INTERNAL_PLUGIN_BASE_URL: "http://workbench:80" - DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo,/plugin/csv" + DEFAULT_PLUGIN_URLS: "/plugin/metamodel,/plugin/model,/plugin/script,/plugin/model-transformation,/plugin/config,/plugin/config-optimization,/plugin/config-mdeo,/plugin/csv,/plugin/model-csv" depends_on: postgres-backend: condition: service_healthy @@ -109,6 +109,7 @@ services: PLUGIN_CONFIG_MDEO_SERVICE: http://service-config-mdeo:3000 BACKEND_SERVICE: http://backend:8080 PLUGIN_CSV_SERVICE: http://service-csv:3000 + PLUGIN_MODEL_CSV_SERVICE: http://service-model-csv:3000 ports: - "4242:80" depends_on: @@ -126,6 +127,8 @@ services: condition: service_healthy service-config-mdeo: condition: service_healthy + service-model-csv: + condition: service_healthy restart: unless-stopped service-metamodel: @@ -187,6 +190,27 @@ services: start_period: 30s restart: unless-stopped + service-model-csv: + build: + context: .. + dockerfile: infra/docker/service-model-csv/Dockerfile + args: + SERVICE_VERSION: ${SERVICE_VERSION:-} + ports: + - "3007:3000" + environment: + PORT: "3000" + HOST: "0.0.0.0" + BACKEND_API_URL: "http://backend:8080/api" + MAX_LANGIUM_INSTANCES: "5" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s + restart: unless-stopped + service-script: build: context: .. diff --git a/infra/docker/service-model-csv/Dockerfile b/infra/docker/service-model-csv/Dockerfile new file mode 100644 index 00000000..cc3293d7 --- /dev/null +++ b/infra/docker/service-model-csv/Dockerfile @@ -0,0 +1,38 @@ +# Build stage +FROM node:24-alpine AS builder + +WORKDIR /build +COPY app/ ./ + +RUN npm ci +RUN npm run build:packages +RUN npm run build:css +RUN npm run -w @mdeo/service-model-csv build:static + +# Production stage +FROM node:24-alpine + +# Install curl for healthchecks +RUN apk add --no-cache curl + +ARG SERVICE_VERSION="" + +WORKDIR /workspace + +# Copy the entire packages structure with built artifacts and dependencies +COPY --from=builder /build/packages /workspace/packages +COPY --from=builder /build/node_modules /workspace/node_modules +COPY --from=builder /build/package.json /workspace/ + +# Set working directory to service-model-csv +WORKDIR /workspace/packages/service-model-csv + +# Set environment variables +ENV NODE_ENV=production +ENV PORT=3000 +ENV HOST=0.0.0.0 +ENV SERVICE_VERSION=${SERVICE_VERSION} + +EXPOSE 3000 + +CMD ["node", "dist/index.js"] diff --git a/infra/docker/workbench/nginx.conf b/infra/docker/workbench/nginx.conf index ee7f7b01..5fe9e003 100644 --- a/infra/docker/workbench/nginx.conf +++ b/infra/docker/workbench/nginx.conf @@ -114,6 +114,20 @@ server { add_header Cross-Origin-Embedder-Policy "require-corp" always; } + # Model-csv plugin + location ^~ /plugin/model-csv/ { + proxy_pass ${PLUGIN_MODEL_CSV_SERVICE}/; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + add_header Cross-Origin-Resource-Policy "cross-origin" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + add_header Cross-Origin-Embedder-Policy "require-corp" always; + } + # Proxy API requests to backend location ^~ /api/ { proxy_pass ${BACKEND_SERVICE}; From 9355be2a78871331e134c61914f4815372a2d419 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:06:23 +0300 Subject: [PATCH 30/41] Add language-model-csv and service-model-csv to package-lock.json npm ci in CI failed because these two new packages, added as part of the contribution-plugin refactor, were never registered in the committed lock file. --- app/package-lock.json | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/app/package-lock.json b/app/package-lock.json index e1cf873e..a50c7d79 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1652,6 +1652,10 @@ "resolved": "packages/language-model", "link": true }, + "node_modules/@mdeo/language-model-csv": { + "resolved": "packages/language-model-csv", + "link": true + }, "node_modules/@mdeo/language-model-transformation": { "resolved": "packages/language-model-transformation", "link": true @@ -1716,6 +1720,10 @@ "resolved": "packages/service-model", "link": true }, + "node_modules/@mdeo/service-model-csv": { + "resolved": "packages/service-model-csv", + "link": true + }, "node_modules/@mdeo/service-model-transformation": { "resolved": "packages/service-model-transformation", "link": true @@ -7927,6 +7935,20 @@ "vscode-languageserver-types": "^3.17.5" } }, + "packages/language-model-csv": { + "name": "@mdeo/language-model-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-metamodel": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-shared": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, "packages/language-model-transformation": { "name": "@mdeo/language-model-transformation", "version": "0.1.0", @@ -8163,6 +8185,7 @@ "@mdeo/editor-model": "^0.1.0", "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", + "@mdeo/language-model-csv": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", @@ -8174,6 +8197,25 @@ "vite": "^7.3.1" } }, + "packages/service-model-csv": { + "name": "@mdeo/service-model-csv", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-common": "^0.1.0", + "@mdeo/language-model": "^0.1.0", + "@mdeo/language-model-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", + "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "lucide": "^0.575.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1", + "tsx": "~4.21.0", + "vite": "~7.3.1" + } + }, "packages/service-model-transformation": { "name": "@mdeo/service-model-transformation", "version": "0.1.0", From 1c775f7a44ec7f405bf9f98a4a097c0247a82678 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:42:21 +0400 Subject: [PATCH 31/41] Add explicit CSV column to property mapping CSV import previously only matched columns to metamodel properties by name. Adds an optional mapping list after the file path: import CSV { Person from "employees.csv" [ "fullName" = name ] } When a mapping is given, only the listed columns are imported (this doubles as the way to skip a column); otherwise name-based matching is unchanged. Unmapped columns or properties in a mapping produce a warning rather than failing the import. Uses square brackets rather than reusing the enclosing block's curly braces, since the grammar's newline-aware brace handling only special-cases "{"/"}"/"("/")" and reusing them caused the parser to misread the mapping list's own opening brace as the start of a new class import. --- .../src/features/csvImport.ts | 77 ++++++++++++++++--- .../src/grammar/csvImportRules.ts | 22 +++++- .../src/grammar/csvImportTypes.ts | 24 +++++- .../src/handler/modelDataHandler.ts | 6 +- 4 files changed, 112 insertions(+), 17 deletions(-) diff --git a/app/packages/language-model-csv/src/features/csvImport.ts b/app/packages/language-model-csv/src/features/csvImport.ts index c72c9c08..55bdd8ff 100644 --- a/app/packages/language-model-csv/src/features/csvImport.ts +++ b/app/packages/language-model-csv/src/features/csvImport.ts @@ -1,8 +1,18 @@ import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "@mdeo/language-model"; +export interface CsvColumnMappingEntry { + csvColumn: string; + property: string; +} + export interface CsvImportEntry { className: string; csvText: string; + /** + * Explicit CSV column to property mappings. When omitted or empty, columns + * are matched to properties by name (the default behavior). + */ + mappings?: CsvColumnMappingEntry[]; } export interface CsvImportResult { @@ -45,11 +55,7 @@ export function importCsvEntries( const dataRows = rows.slice(1); const idColIndex = header.indexOf("_id"); - const propertyNames = new Set(classInfo.properties.map(p => p.name)); - const unknownCols = header.filter(h => h !== "_id" && !propertyNames.has(h)); - if (unknownCols.length > 0) { - warnings.push(`CSV for '${entry.className}' has unknown columns: ${unknownCols.join(", ")} — they will be ignored.`); - } + const columnToProperty = resolveColumnMapping(entry, header, classInfo, warnings); dataRows.forEach((row, rowIndex) => { const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); @@ -65,9 +71,11 @@ export function importCsvEntries( const properties: Record = {}; header.forEach((colName, colIndex) => { if (colName === "_id") return; - const prop = classInfo.properties.find(p => p.name === colName); + const propName = columnToProperty.get(colName); + if (!propName) return; + const prop = classInfo.properties.find(p => p.name === propName); if (!prop) return; - properties[colName] = convertCellValue(normalizedRow[colIndex], prop); + properties[propName] = convertCellValue(normalizedRow[colIndex], prop); }); allInstances.push({ @@ -87,9 +95,11 @@ export function importCsvEntries( const header = rows[0]; const dataRows = rows.slice(1); + const columnToProperty = resolveColumnMapping(entry, header, classInfo, []); const refCols = header.filter(colName => { - const prop = classInfo.properties.find(p => p.name === colName); + const propName = columnToProperty.get(colName); + const prop = propName ? classInfo.properties.find(p => p.name === propName) : undefined; return prop?.isReference; }); @@ -100,7 +110,8 @@ export function importCsvEntries( const sourceInstanceName = `${entry.className}_${rowIndex}`; refCols.forEach(colName => { - const prop = classInfo.properties.find(p => p.name === colName)!; + const propName = columnToProperty.get(colName)!; + const prop = classInfo.properties.find(p => p.name === propName)!; const rawValue = normalizedRow[header.indexOf(colName)]; if (!rawValue) return; @@ -113,7 +124,7 @@ export function importCsvEntries( } allLinks.push({ sourceName: sourceInstanceName, - sourceProperty: colName, + sourceProperty: propName, targetName: targetInstanceName, targetProperty: null }); @@ -125,6 +136,52 @@ export function importCsvEntries( return { instances: allInstances, links: allLinks, warnings }; } +/** + * Resolves which CSV column maps to which metamodel property for one entry. + * + * When the entry has an explicit mapping, only the mapped columns are included + * (this is also how a column can be intentionally skipped). Otherwise, columns + * are matched to properties by name, and unmatched columns are warned about. + * + * @param entry The import entry, whose optional `mappings` take precedence + * @param header The CSV header row + * @param classInfo The metamodel class the CSV is imported into + * @param warnings Warnings accumulator; mutated in place + * @returns A map from CSV column name to metamodel property name + */ +function resolveColumnMapping( + entry: CsvImportEntry, + header: string[], + classInfo: MetamodelClassInfo, + warnings: string[] +): Map { + if (entry.mappings != undefined && entry.mappings.length > 0) { + const columnToProperty = new Map(); + for (const mapping of entry.mappings) { + if (!header.includes(mapping.csvColumn)) { + warnings.push(`CSV for '${entry.className}' has no column '${mapping.csvColumn}' referenced by its mapping — skipping.`); + continue; + } + if (!classInfo.properties.some(p => p.name === mapping.property)) { + warnings.push(`Class '${entry.className}' has no property '${mapping.property}' referenced by its mapping — skipping.`); + continue; + } + columnToProperty.set(mapping.csvColumn, mapping.property); + } + return columnToProperty; + } + + const propertyNames = new Set(classInfo.properties.map(p => p.name)); + const columnToProperty = new Map( + header.filter(h => h !== "_id" && propertyNames.has(h)).map(h => [h, h]) + ); + const unknownCols = header.filter(h => h !== "_id" && !columnToProperty.has(h)); + if (unknownCols.length > 0) { + warnings.push(`CSV for '${entry.className}' has unknown columns: ${unknownCols.join(", ")} — they will be ignored.`); + } + return columnToProperty; +} + export interface MetamodelPropertyInfo { name: string; type: "string" | "int" | "long" | "double" | "float" | "boolean" | "enum" | "reference"; diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts index af3ba60e..dd3b1549 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportRules.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -1,5 +1,5 @@ -import { createRule, many, or, ref, createExternalTerminalRule } from "@mdeo/language-common"; -import { CsvClassImport, CsvImportBlock, ExternalClass } from "./csvImportTypes.js"; +import { createRule, many, optional, or, ref, createExternalTerminalRule } from "@mdeo/language-common"; +import { CsvClassImport, CsvColumnMapping, CsvImportBlock, ExternalClass } from "./csvImportTypes.js"; /** * Stand-ins for the base language's common terminals, so this grammar's @@ -11,9 +11,25 @@ const ID = createExternalTerminalRule("ID"); const STRING = createExternalTerminalRule("STRING"); const NEWLINE = createExternalTerminalRule("NEWLINE"); +export const CsvColumnMappingRule = createRule("CsvColumnMappingRule") + .returns(CsvColumnMapping) + .as(({ set }) => [set("csvColumn", STRING), "=", set("property", ID)]); + +/** + * The optional explicit mapping list uses square brackets rather than curly + * braces, since curly braces already close the enclosing `import CSV { }` + * block and this grammar's newline-aware brace handling only special-cases + * "{"/"}"/"("/")" - reusing braces here caused the parser to misread the + * mapping list's own opening brace as the start of a new class import. + */ export const CsvClassImportRule = createRule("CsvClassImportRule") .returns(CsvClassImport) - .as(({ set }) => [set("class", ref(ExternalClass, ID)), "from", set("file", STRING)]); + .as(({ set, add }) => [ + set("class", ref(ExternalClass, ID)), + "from", + set("file", STRING), + optional("[", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "]") + ]); /** * The content of a CSV import block (everything between the braces). diff --git a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts index e55ce295..e14f618a 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts @@ -9,14 +9,32 @@ import type { ClassType } from "@mdeo/language-metamodel"; */ export const ExternalClass = createExternalInterface("Class"); +/** + * An explicit CSV column to metamodel property mapping. + * Format: "csvColumnName" = modelPropertyName + */ +export const CsvColumnMapping = createInterface("CsvColumnMapping").attrs({ + csvColumn: String, + property: String +}); + +export type CsvColumnMappingType = ASTType; + /** * A single CSV class import entry. - * Maps a metamodel class to a CSV file. - * Format: ClassName from "path/to/file.csv" + * Maps a metamodel class to a CSV file, with an optional explicit column mapping. + * Format: + * ClassName from "path/to/file.csv" + * or, with an explicit mapping: + * ClassName from "path/to/file.csv" [ + * "csvColumnName" = modelPropertyName + * ] + * When no mapping list is given, columns are matched to properties by name. */ export const CsvClassImport = createInterface("CsvClassImport").attrs({ class: Ref(() => ExternalClass), - file: String + file: String, + mappings: [CsvColumnMapping] }); export type CsvClassImportType = ASTType; diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index 25675c8e..0f1adda0 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -86,7 +86,11 @@ export const modelDataHandler: FileDataHandler const { content } = await serverApi.readFile(csvPath); return { className: entry.class?.ref?.name ?? "", - csvText: content + csvText: content, + mappings: (entry.mappings ?? []).map((mapping: any) => ({ + csvColumn: mapping.csvColumn, + property: mapping.property + })) }; }) ); From 4653e59a6898054aba46929dace33ce701059db4 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Wed, 22 Jul 2026 20:46:40 +0400 Subject: [PATCH 32/41] Wait for service-csv health before starting workbench workbench proxies /plugin/csv/ to service-csv but was missing it from depends_on, unlike every other plugin service. Could cause startup-time 502s or a missing plugin manifest while service-csv was still starting. --- infra/docker-compose-dev.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/docker-compose-dev.yaml b/infra/docker-compose-dev.yaml index 109886e8..8681056c 100644 --- a/infra/docker-compose-dev.yaml +++ b/infra/docker-compose-dev.yaml @@ -129,6 +129,8 @@ services: condition: service_healthy service-model-csv: condition: service_healthy + service-csv: + condition: service_healthy restart: unless-stopped service-metamodel: From eddadd66540c5b538823e7d08c44cf89590f679c Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:13:14 +0400 Subject: [PATCH 33/41] Address Copilot review comments on CSV import PR - nginx: /plugin/csv/ location was missing the proxy_http_version, forwarded headers, and cross-origin headers every other plugin location has, breaking ESM module loading and cross-origin isolation for the CSV plugin specifically. - service-model: register the generated model language (.m_gen) that was dropped when this service was rewritten for the CSV contribution plugin refactor, even though the platform still produces .m_gen files (e.g. optimizer solutions) and language-model still has full support for it. - modelDataHandler: a CSV import present in a .m file was replacing all hand-authored objects/links instead of merging with them, so mixing manually-authored data with CSV-imported data in the same file silently lost the manual entries. - Moved the quote-aware CSV row/field parser out of language-model-csv (used by the backend import) into language-shared so language-model's live diagram rendering (modelGModelFactory) can use the same correct parser instead of a naive split(",")/split("\n") that miscounts rows when a field contains a quoted comma or newline. Verified live: mixed manual + CSV objects now render together in the diagram (confirmed with a hand-authored Company instance alongside CSV-imported Person rows), .m_gen is now registered in the plugin manifest, and the /plugin/csv/ headers match the other plugin locations. --- .../src/features/csvImport.ts | 56 +---------------- .../diagram-server/modelGModelFactory.ts | 5 +- app/packages/language-shared/src/index.ts | 1 + app/packages/language-shared/src/util/csv.ts | 63 +++++++++++++++++++ .../src/handler/modelDataHandler.ts | 7 ++- app/packages/service-model/src/index.ts | 40 ++++++++++-- infra/docker/workbench/nginx.conf | 8 +++ 7 files changed, 116 insertions(+), 64 deletions(-) create mode 100644 app/packages/language-shared/src/util/csv.ts diff --git a/app/packages/language-model-csv/src/features/csvImport.ts b/app/packages/language-model-csv/src/features/csvImport.ts index c72c9c08..0a571b00 100644 --- a/app/packages/language-model-csv/src/features/csvImport.ts +++ b/app/packages/language-model-csv/src/features/csvImport.ts @@ -1,4 +1,5 @@ import type { ModelData, ModelDataInstance, ModelDataPropertyValue } from "@mdeo/language-model"; +import { parseCsv } from "@mdeo/language-shared"; export interface CsvImportEntry { className: string; @@ -171,58 +172,3 @@ function normalizeRow(row: string[], expectedLength: number, rowNumber: number, return row.slice(0, expectedLength); } -function parseCsv(text: string): string[][] { - const rows: string[][] = []; - let currentRow: string[] = []; - let currentField = ""; - let inQuotes = false; - let i = 0; - - const endField = () => { - currentRow.push(currentField); - currentField = ""; - }; - - const endRow = () => { - endField(); - if (!(currentRow.length === 1 && currentRow[0].trim() === "")) { - rows.push(currentRow); - } - currentRow = []; - }; - - while (i < text.length) { - const c = text[i]; - if (inQuotes) { - if (c === '"') { - if (i + 1 < text.length && text[i + 1] === '"') { - currentField += '"'; - i++; - } else { - inQuotes = false; - } - } else { - currentField += c; - } - } else { - if (c === '"') { - inQuotes = true; - } else if (c === ',') { - endField(); - } else if (c === '\r') { - // skip - } else if (c === '\n') { - endRow(); - } else { - currentField += c; - } - } - i++; - } - - if (currentField !== "" || currentRow.length > 0) { - endRow(); - } - - return rows; -} diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 65df80f0..6a6244ad 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -7,7 +7,8 @@ import { EdgeLayoutMetadataUtil, NodeLayoutMetadataUtil, parseIdentifier, - resolveRelativePath + resolveRelativePath, + parseCsv } from "@mdeo/language-shared"; import type { ModelIdRegistry, GraphMetadata } from "@mdeo/language-shared"; import type { NodeLayoutMetadata, EdgeLayoutMetadata } from "@mdeo/protocol-common"; @@ -496,7 +497,7 @@ export class ModelGModelFactory extends BaseGModelFactory { try { const uri = resolveRelativePath(doc, entry.file ?? ""); const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); - const rows = csvContent.split(/\r?\n/).filter((line: string) => line.trim() !== "").map((line: string) => line.split(",")); + const rows = parseCsv(csvContent); if (rows.length < 2) continue; const classHierarchy = resolveClassChain(classRef, this.reflection).map((c) => c.name); rows.slice(1).forEach((_row: string[], rowIndex: number) => { diff --git a/app/packages/language-shared/src/index.ts b/app/packages/language-shared/src/index.ts index 81dd82c0..39497703 100644 --- a/app/packages/language-shared/src/index.ts +++ b/app/packages/language-shared/src/index.ts @@ -79,6 +79,7 @@ export * from "./serialization/comments.js"; export * from "./serialization/defaultTokenSerializers.js"; export * from "./serialization/indentationGuesser.js"; export * from "./serialization/newlineSepSerializer.js"; +export * from "./util/csv.js"; export * from "./util/pathUtils.js"; export * from "./workspace/workspaceEditService.js"; export * from "./sharedImport.js"; diff --git a/app/packages/language-shared/src/util/csv.ts b/app/packages/language-shared/src/util/csv.ts new file mode 100644 index 00000000..cdf698f4 --- /dev/null +++ b/app/packages/language-shared/src/util/csv.ts @@ -0,0 +1,63 @@ +/** + * Parses CSV text into rows of fields, honouring RFC 4180 quoting: quoted fields + * may contain commas and newlines, and `""` inside a quoted field is an escaped + * literal quote. Blank lines are skipped. + * + * @param text The raw CSV file content + * @returns The parsed rows, each an array of field values + */ +export function parseCsv(text: string): string[][] { + const rows: string[][] = []; + let currentRow: string[] = []; + let currentField = ""; + let inQuotes = false; + let i = 0; + + const endField = () => { + currentRow.push(currentField); + currentField = ""; + }; + + const endRow = () => { + endField(); + if (!(currentRow.length === 1 && currentRow[0].trim() === "")) { + rows.push(currentRow); + } + currentRow = []; + }; + + while (i < text.length) { + const c = text[i]; + if (inQuotes) { + if (c === '"') { + if (i + 1 < text.length && text[i + 1] === '"') { + currentField += '"'; + i++; + } else { + inQuotes = false; + } + } else { + currentField += c; + } + } else { + if (c === '"') { + inQuotes = true; + } else if (c === ",") { + endField(); + } else if (c === "\r") { + // skip + } else if (c === "\n") { + endRow(); + } else { + currentField += c; + } + } + i++; + } + + if (currentField !== "" || currentRow.length > 0) { + endRow(); + } + + return rows; +} diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index 25675c8e..ed5316aa 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -93,10 +93,13 @@ export const modelDataHandler: FileDataHandler const result = importCsvEntries(csvEntries, metamodelClasses); + const converter = new ModelDataConverter(reflection); + const handAuthored = converter.convertModel(model); + const modelData: ModelData = { metamodelPath, - instances: result.instances, - links: result.links + instances: [...handAuthored.instances, ...result.instances], + links: [...handAuthored.links, ...result.links] }; return { diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index 9771ccd8..e0fa39ee 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -15,7 +15,7 @@ import { astHandler, AST_HANDLER_KEY } from "@mdeo/service-common"; -import type { ModelServices } from "@mdeo/language-model"; +import type { ModelServices, GeneratedModelServices } from "@mdeo/language-model"; import type { LanguagePlugin } from "@mdeo/plugin"; const modelLanguagePlugin: LanguagePlugin = { @@ -41,9 +41,31 @@ const modelLanguagePlugin: LanguagePlugin = { isGenerated: false }; +/** + * Language plugin definition for the generated model language (.m_gen), e.g. optimizer solutions. + */ +const generatedModelLanguagePlugin: LanguagePlugin = { + id: "model_gen", + name: "Generated Model", + extension: ".m_gen", + newFileAction: false, + icon: convertIcon(BookOpen), + serverPlugin: { + import: "generatedLanguage.js" + }, + graphicalEditorPlugin: { + import: "editor.js", + stylesUrl: "styles.css", + stylesCls: "editor-model" + }, + textualEditorPlugin: undefined, + isGenerated: true +}; + initializePluginContext(); const { modelPluginProvider } = await import("@mdeo/language-model"); +const { generatedModelPluginProvider } = await import("@mdeo/language-model"); const { modelDataHandler, MODEL_DATA_HANDLER_KEY } = await import("./handler/modelDataHandler.js"); const envConfig = parseServiceConfigFromEnv(); @@ -51,9 +73,9 @@ const envConfig = parseServiceConfigFromEnv(); const modelServicePlugin: ServicePluginDefinition = { id: "model-service", name: "Model", - description: "Language support for model definitions (.m files)", + description: "Language support for model definitions (.m and .m_gen files)", icon: convertIcon(BookOpen), - languagePlugins: [modelLanguagePlugin], + languagePlugins: [modelLanguagePlugin, generatedModelLanguagePlugin], contributionPlugins: [] }; @@ -66,10 +88,18 @@ const modelLanguageConfig: LanguageServiceConfig = { } }; -const config: ServiceConfig = { +const generatedModelLanguageConfig: LanguageServiceConfig = { + languagePlugin: generatedModelLanguagePlugin, + languagePluginProvider: generatedModelPluginProvider, + fileDataHandlers: { + [AST_HANDLER_KEY]: astHandler + } +}; + +const config: ServiceConfig = { ...envConfig, plugin: modelServicePlugin, - languages: [modelLanguageConfig] + languages: [modelLanguageConfig, generatedModelLanguageConfig] }; await startLanguageService(config); diff --git a/infra/docker/workbench/nginx.conf b/infra/docker/workbench/nginx.conf index 5fe9e003..88fb8aff 100644 --- a/infra/docker/workbench/nginx.conf +++ b/infra/docker/workbench/nginx.conf @@ -52,10 +52,18 @@ server { add_header Cross-Origin-Embedder-Policy "require-corp" always; } + # CSV plugin location ^~ /plugin/csv/ { proxy_pass ${PLUGIN_CSV_SERVICE}/; + proxy_http_version 1.1; + proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + add_header Cross-Origin-Resource-Policy "cross-origin" always; + add_header Cross-Origin-Opener-Policy "same-origin" always; + add_header Cross-Origin-Embedder-Policy "require-corp" always; } # Model-transformation plugin From df6dc0a766475e6a8d0a32b194b42006c296fceb Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sat, 25 Jul 2026 00:53:47 +0400 Subject: [PATCH 34/41] Use curly braces with a keyword for explicit CSV column mapping Niklas asked to avoid square brackets and instead follow how other DSLs in this codebase disambiguate two adjacent brace blocks in one rule: an explicit keyword between them, not a different bracket type (e.g. model-transformation's match { pattern } then { block }). The explicit mapping list now reads: Person from "file.csv" with { "column" = property } instead of the previous square-bracket form. Verified live: two CSV imports in the same block, the first with an explicit mapping, parse correctly now with no ambiguity errors. --- .../src/grammar/csvImportRules.ts | 18 +++++++++++------- app/packages/service-model-csv/src/index.ts | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts index dd3b1549..5d05e790 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportRules.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -1,4 +1,4 @@ -import { createRule, many, optional, or, ref, createExternalTerminalRule } from "@mdeo/language-common"; +import { createRule, many, optional, or, ref, group, createExternalTerminalRule } from "@mdeo/language-common"; import { CsvClassImport, CsvColumnMapping, CsvImportBlock, ExternalClass } from "./csvImportTypes.js"; /** @@ -16,11 +16,13 @@ export const CsvColumnMappingRule = createRule("CsvColumnMappingRule") .as(({ set }) => [set("csvColumn", STRING), "=", set("property", ID)]); /** - * The optional explicit mapping list uses square brackets rather than curly - * braces, since curly braces already close the enclosing `import CSV { }` - * block and this grammar's newline-aware brace handling only special-cases - * "{"/"}"/"("/")" - reusing braces here caused the parser to misread the - * mapping list's own opening brace as the start of a new class import. + * The optional explicit mapping list is introduced by the "with" keyword, + * the same way the model-transformation grammar disambiguates two adjacent + * brace blocks in one rule (e.g. `match { pattern } then { block }`): the + * keyword between them is what tells the parser which block it's entering, + * not the bracket type. Reusing bare curly braces here, with nothing between + * `file` and the mapping list's own "{", caused the parser to misread the + * mapping list's opening brace as the start of a new class import. */ export const CsvClassImportRule = createRule("CsvClassImportRule") .returns(CsvClassImport) @@ -28,7 +30,9 @@ export const CsvClassImportRule = createRule("CsvClassImportRule") set("class", ref(ExternalClass, ID)), "from", set("file", STRING), - optional("[", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "]") + optional( + group("with", "{", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "}") + ) ]); /** diff --git a/app/packages/service-model-csv/src/index.ts b/app/packages/service-model-csv/src/index.ts index f7aa15ac..853dab6f 100644 --- a/app/packages/service-model-csv/src/index.ts +++ b/app/packages/service-model-csv/src/index.ts @@ -49,8 +49,8 @@ const modelCsvServicePlugin: ServicePluginDefinition = { contributionPlugins: [ { languageId: "model", - description: "Provides import CSV { ClassName from \"file.csv\" } syntax for the model language", - additionalKeywords: ["import", "CSV", "from"], + description: "Provides import CSV { ClassName from \"file.csv\" with { column = property } } syntax for the model language", + additionalKeywords: ["import", "CSV", "from", "with"], serverContributionPlugins: [createModelCsvContributionPlugin()] } ] From a380d1d9e2baf00223c605aaf1025fa184b8e5c0 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:08:17 +0400 Subject: [PATCH 35/41] Show property values on CSV-imported diagram nodes createCsvNodes only emitted a header label, so a CSV-imported instance rendered as a bare "Company_0 : Company" box with none of its data, while hand-authored objects showed their property assignments. The row data was read and then discarded. Columns are now matched to properties by name across the class' extension chain and rendered the same way hand-authored values are, using the same type interpretation the import itself applies so the diagram agrees with the imported result. Labels are read-only since edits could not be written back to the CSV file. --- .../diagram-server/modelGModelFactory.ts | 155 +++++++++++++++++- 1 file changed, 151 insertions(+), 4 deletions(-) diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index 6a6244ad..da2309c1 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -13,7 +13,7 @@ import { import type { ModelIdRegistry, GraphMetadata } from "@mdeo/language-shared"; import type { NodeLayoutMetadata, EdgeLayoutMetadata } from "@mdeo/protocol-common"; import { ID } from "@mdeo/language-common"; -import { resolveClassChain, type ClassType } from "@mdeo/language-metamodel"; +import { resolveClassChain, type ClassType, type PropertyType } from "@mdeo/language-metamodel"; import type { PartialModel, PartialObjectInstance, @@ -42,6 +42,20 @@ const { GGraph } = sharedImport("@eclipse-glsp/server"); type GGraphType = ReturnType["proxy"]; +/** + * Reserved CSV column holding a row's identifier for cross-object references. + * It is not a metamodel property, so it is never displayed as one. + */ +const CSV_ID_COLUMN = "_id"; + +/** + * A CSV column that maps onto a metamodel property of the imported class. + */ +interface CsvColumnBinding { + index: number; + property: PropertyType; +} + /** * Factory for creating GLSP graph models from model AST. * Transforms the model source into a visual graph representation @@ -499,8 +513,10 @@ export class ModelGModelFactory extends BaseGModelFactory { const csvContent = await this.modelState.languageServices.shared.workspace.FileSystemProvider.readFile(uri); const rows = parseCsv(csvContent); if (rows.length < 2) continue; - const classHierarchy = resolveClassChain(classRef, this.reflection).map((c) => c.name); - rows.slice(1).forEach((_row: string[], rowIndex: number) => { + const classChain = resolveClassChain(classRef, this.reflection); + const classHierarchy = classChain.map((c) => c.name); + const columns = this.resolveCsvColumns(rows[0], classChain); + rows.slice(1).forEach((row: string[], rowIndex: number) => { const instanceName = `${classRef.name}_${rowIndex}`; const nodeId = `csv-node-${nodeIndex++}`; const metadata = validatedMetadata.nodes[nodeId]?.meta ?? {}; @@ -511,7 +527,10 @@ export class ModelGModelFactory extends BaseGModelFactory { .classHierarchy(classHierarchy) .meta(metadata) .build(); - node.children.push(...this.createObjectHeader(nodeId, instanceName, classRef.name)); + node.children.push( + ...this.createObjectHeader(nodeId, instanceName, classRef.name), + ...this.createCsvPropertyAssignments(nodeId, columns, row) + ); graph.children.push(node); }); } catch { @@ -520,4 +539,132 @@ export class ModelGModelFactory extends BaseGModelFactory { } } + /** + * Resolves which CSV columns correspond to properties of the imported class. + * + * Columns are matched to properties by name across the class' whole extension + * chain, mirroring how the import itself resolves them. Unmatched columns and + * the reserved `_id` row identifier are dropped, so they are simply not shown. + * + * @param header The CSV header row + * @param classChain The imported class and its extended classes + * @returns The column index and property for every column that maps to one + */ + private resolveCsvColumns(header: string[], classChain: ClassType[]): CsvColumnBinding[] { + const propertiesByName = new Map(); + for (const cls of classChain) { + for (const property of cls.properties ?? []) { + if (property?.name != undefined && !propertiesByName.has(property.name)) { + propertiesByName.set(property.name, property); + } + } + } + + const columns: CsvColumnBinding[] = []; + header.forEach((columnName, index) => { + if (columnName === CSV_ID_COLUMN) { + return; + } + const property = propertiesByName.get(columnName); + if (property != undefined) { + columns.push({ index, property }); + } + }); + return columns; + } + + /** + * Creates the property assignments compartment for a CSV-imported node. + * + * Mirrors {@link createPropertyAssignments}, but reads values straight from a + * CSV row instead of from property assignment AST nodes, since CSV-imported + * instances have no backing AST. The labels are read-only because editing them + * could not be written back to the CSV file. + * + * @param nodeId The ID of the object node + * @param columns The columns that map to properties, from {@link resolveCsvColumns} + * @param row The CSV data row for this node + * @returns Array of GModelElements for the properties compartment + */ + private createCsvPropertyAssignments( + nodeId: string, + columns: CsvColumnBinding[], + row: string[] + ): GModelElement[] { + const propertyLabels: GPropertyLabel[] = []; + for (const { index, property } of columns) { + const rawValue = row[index]; + if (rawValue == undefined || rawValue.trim() === "") { + continue; + } + const propertyName = this.modelState.languageServices.AstSerializer.serializePrimitive( + { value: property.name }, + ID + ); + propertyLabels.push( + GPropertyLabel.builder() + .id(`${nodeId}__prop-${index}`) + .text(`${propertyName} = ${this.formatCsvValue(rawValue, property)}`) + .readonly(true) + .build() + ); + } + + if (propertyLabels.length === 0) { + return []; + } + + const divider = GHorizontalDivider.builder().type(ModelElementType.DIVIDER).id(`${nodeId}__divider`).build(); + const propertiesCompartment = GCompartment.builder() + .type(ModelElementType.COMPARTMENT) + .id(`${nodeId}__properties-compartment`) + .build(); + propertiesCompartment.children.push(...propertyLabels); + + return [divider, propertiesCompartment]; + } + + /** + * Formats a raw CSV cell for display, following the property's declared type. + * + * Uses the same type interpretation as the CSV import itself, so the diagram + * shows the value the import would produce, and renders it the same way + * hand-authored values are rendered. Values that don't parse as their declared + * numeric type fall back to being shown as text rather than as `NaN`. + * + * @param rawValue The raw cell text + * @param property The metamodel property the column maps to + * @returns The formatted value string + */ + private formatCsvValue(rawValue: string, property: PropertyType): string { + const value = rawValue.trim(); + const propertyType = property.type as + | { name?: string; enum?: { ref?: { name?: string }; $refText?: string } } + | undefined; + + const enumName = + (propertyType?.enum?.ref as { name?: string } | undefined)?.name ?? + (propertyType?.enum != undefined ? parseIdentifier(propertyType.enum.$refText ?? "?") : undefined); + if (enumName != undefined) { + const serializer = this.modelState.languageServices.AstSerializer; + return `${serializer.serializePrimitive({ value: enumName }, ID)}.${serializer.serializePrimitive({ value }, ID)}`; + } + + switch (propertyType?.name) { + case "int": + case "long": { + const parsed = parseInt(value, 10); + return isNaN(parsed) ? `"${value}"` : String(parsed); + } + case "double": + case "float": { + const parsed = parseFloat(value); + return isNaN(parsed) ? `"${value}"` : String(parsed); + } + case "boolean": + return String(value.toLowerCase() === "true"); + default: + return `"${value}"`; + } + } } From 5750a85b94501c34ffbca015999fd1c0dfe2d9cc Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:10:33 +0400 Subject: [PATCH 36/41] Restore doc comments in modelTypes.ts and untrack .DS_Store files The contribution-plugin refactor stripped every doc comment out of modelTypes.ts, which was unintentional collateral rather than a deliberate change. Restored them and added comments for the two new declarations. Also removes infra/.DS_Store and infra/docker/.DS_Store from tracking. They match .gitignore already, so they were committed before the ignore rule applied to them. --- .../language-model/src/grammar/modelTypes.ts | 77 ++++++++++++++++++ infra/.DS_Store | Bin 8196 -> 0 bytes infra/docker/.DS_Store | Bin 8196 -> 0 bytes 3 files changed, 77 insertions(+) delete mode 100644 infra/.DS_Store delete mode 100644 infra/docker/.DS_Store diff --git a/app/packages/language-model/src/grammar/modelTypes.ts b/app/packages/language-model/src/grammar/modelTypes.ts index 40e79ca0..feecaec1 100644 --- a/app/packages/language-model/src/grammar/modelTypes.ts +++ b/app/packages/language-model/src/grammar/modelTypes.ts @@ -2,68 +2,139 @@ import { createInterface, createType, Optional, Ref, type ASTType, type BaseType import { AssociationEnd, Class, Enum, EnumEntry, Property } from "@mdeo/language-metamodel"; import type { AstNode } from "langium"; +/** + * Simple value for property assignments. + * Supports string, number, and boolean literals. + */ export const SimpleValue = createInterface("SimpleValue").attrs({ stringValue: Optional(String), numberValue: Optional(Number), booleanValue: Optional(Boolean) }); +/** + * Type representing a SimpleValue AST node. + */ export type SimpleValueType = ASTType; +/** + * Enum value for property assignments. + * References an enum and one of its entries from the metamodel. + * Written as EnumName.Entry in the model language. + */ export const EnumValue = createInterface("EnumValue").attrs({ enumRef: Ref(() => Enum), value: Ref(() => EnumEntry) }); +/** + * Type representing an EnumValue AST node. + */ export type EnumValueType = ASTType; +/** + * Single value type union. + * Represents either a simple value or an enum value. + */ export const SingleValue: BaseType = createType("SingleValue").types(SimpleValue, EnumValue); +/** + * Type representing a SingleValue AST node. + */ export type SingleValueType = ASTType; +/** + * List value for property assignments with multiplicity > 1. + * Contains a comma-separated list of values in square brackets. + */ export const ListValue = createInterface("ListValue").attrs({ values: [SingleValue] }); +/** + * Type representing a ListValue AST node. + */ export type ListValueType = ASTType; +/** + * Literal value type union for property assignments. + * Can be a single value (simple or enum) or a list value. + */ export const LiteralValue: BaseType = createType("LiteralValue").types(SimpleValue, EnumValue, ListValue); +/** + * Type representing a LiteralValue AST node. + */ export type LiteralValueType = ASTType; +/** + * Property assignment in an object instance. + * References a property from the class and assigns it a value. + */ export const PropertyAssignment = createInterface("PropertyAssignment").attrs({ name: Ref(() => Property), value: LiteralValue }); +/** + * Type representing a PropertyAssignment AST node. + */ export type PropertyAssignmentType = ASTType; +/** + * Object instance definition. + * Represents an instance of a class from the metamodel. + */ export const ObjectInstance = createInterface("ObjectInstance").attrs({ name: String, class: Ref(() => Class), properties: [PropertyAssignment] }); +/** + * Type representing an ObjectInstance AST node. + */ export type ObjectInstanceType = ASTType; +/** + * Link end representing one side of a link. + * References an object instance with an optional property specification. + */ export const LinkEnd = createInterface("LinkEnd").attrs({ object: Ref(() => ObjectInstance), property: Optional(Ref(() => AssociationEnd)) }); +/** + * Type representing a LinkEnd AST node. + */ export type LinkEndType = ASTType; +/** + * Link definition between two object instances. + * Format: objectId1[.property] -- objectId2[.property] + */ export const Link = createInterface("Link").attrs({ source: LinkEnd, target: LinkEnd }); +/** + * Type representing a Link AST node. + */ export type LinkType = ASTType; +/** + * Metamodel file import. + * Simple import statement for referencing metamodel files. + */ export const MetamodelFileImport = createInterface("MetamodelFileImport").attrs({ file: String }); +/** + * Type representing MetamodelFileImport AST node. + */ export type MetamodelFileImportType = ASTType; /** @@ -73,6 +144,9 @@ export type MetamodelFileImportType = ASTType; */ export const BaseModelImport = createInterface("BaseModelImport").attrs({}); +/** + * Type representing a BaseModelImport AST node. + */ export type BaseModelImportType = ASTType; /** @@ -87,4 +161,7 @@ export const Model = createInterface("Model").attrs({ links: [Link] }); +/** + * Model AST type. + */ export type ModelType = ASTType; diff --git a/infra/.DS_Store b/infra/.DS_Store deleted file mode 100644 index 5e79992e6df7f2ec950fb3c6c5cc8bb917496f18..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMYitx%6u#fIlo>m~v=%7HWVbG_;9?&Xq#(Go?<9Xp+=JF{CJ zsVOl&FeVybiNA=?pN$#~8iS(0OngPtkVqmSY9jve7k@ATJu`Qf*oB&y7y~*tnS17( zd(N3TXTNiA&y+ES*1XopSS@2rq|2j9B~8~E-nw^PGlW1=P84L%n9g#UF=sSm+L>## zLqV8I?}^!#T`An1O3E;FV7W)0oY&Y`CO{8;_Qy#vGMh-Dw=Q6Sg}*qoB6 zo+=H=bP8?!lSEmv7}D8dNYX8p@*K4%)!Qfcr;My^C5^pr_9&&lR&)7)b_hHKbXZ_3pVxkX3QmQsC=opalKX9J3o zcl5nF`7#Q6J9#8m)!x%~k9uyLeR|Fz8cQ_r5T(hv3nNQcu5P;P?!?xeo%c`8Q>v=v z8hJra!ElU>sdtYVnmg2+cCxyq8P;GQCD(F|gNB}yst}jDtn`RJRb9Ji@oh^~wN$(4 zC6&(@59bZ{uo4qR{){>`CMoZU{TjVwr>%J&HGf5|Q&))Meox+((`n{nd6gQMl(g98 zP=dINdbv?j9I0!V5Wr2F<#nQ>N!?~TqnljRB5xFxT~behN6v*c@;#!I%Gx6%6pX`V zt!lfdd?r5NJ-8$7xVuMntN2Q->JCxtbB5A6efJpQuV~RBivABva?fO|X1I3F&&GXq zs@m(h+M;Q?MnNyGT~w_m)oxMwmTVNmDj0-#`Ykx{#aIz@vBWBV8#_IG-oNv6EY%bh z0_vS_DqGIh(|Oj#4pO}xV^6aa>|J(>eZ;tqZ&)F6su8> z1~g&=HewStV;8#6jeW>q7$z*(a50WYFo9!u9M9lcJcsA;GG4`Ncmr=@5-0IKKES8= z9ADroe2sJX85i&iF5)t-;CG=)SS&0RmI=#+dZ9sR64nUYgzdr}p;yQVgTfGxKq!{m zf$5P6hl!uIgHX8zUjT*4>z7+(`;MLW${ja+xqUa+E3jK@<}X+nUAC&edHtp>S7FFe zTLkK#3&6+xv`X zJI^k#i|i6APzePQti+vYCS-5J4nlVaItkq=^kF~xp<@sR9wMx}DButd;|LzbV|W5j z;y5Aw1-yuta00Io+FvKUPvR}Sjd$=NPT^ym#wR}Rr|>;~z>g(J?wpQfTr5R$A(^vG z+ZrT!5&hFmZf>rOzkaRP|8Jf7`~S@yWcc=B2Eq*dWd=~*o@{R=m6FPWS_>3pvRlh7xY!E?DF`lYxk)R?wsc#HbuU9Zb~;mcX182Y zQ)9efOf=pSpNQATMvVrILD44@Z_zX)l1PY}h%Y|zf(ht<=FAdW+7~qjbWU>4fBy4d zX8wP_|C~L$j4?Flv-ONsF~(%-oN8q>Ow+iW_v@MztdtT3*)x`A2D4d)4Wvh|(+(HH z41^g7GZ1DV%s`ld8zBR9X7eV`aPA9fIEEPrGw{F60N)>?)H#g@bWBix>!88E0ubd_ z0Kd_k?g73{G@$W-jtT0Ul237aKwygCiUDCx@+em)8V~50pu(I%m@@=VMsP!c|8%m8 z`sxgELBlc3K$wB+Gr-G7VI$08IW~Ch{oNf-TTYs!C)2~0>3Vf_XCRf7&YC?(o+FpZ z2NMI%NWx2bdAB*0JHWXP$Fb7Ic`Ci%G)DTg$`;4=Ql@R>d;>FUkV?g$aap4SP<+7 zZ0?lUo}NB1%Pq8sz)BLuz|Ix}Q*J3!=jvU_?jE%_X=WTdVIDR5wD~SYw~N@lIio;P zb!8kYKV-XoT6rdAW%5?aGqze5SEjZ3ZYJkg)?UXoJ=3wflb&(dD>|AaO7^%;&TH|V z^=Vq(HTD_g%PbhJ@1K~j zl~<^h>cXyq>6&TF=o~S#-avQC%@}spwEKG~#kOZ2GL4*4j+oM6r-qD)imJs+Zd8sWDBAmSZnmmX0AFv(88nJzb+sN- zw3OW8Qi6nwTD4x$T%}`>a3D+@)%CKLRXVLy+OWh@le$UPb}L;45xH2bRPT|MWX2g9 zqF`LH)~vV6+UN2E{K0K0*V{8}*u__B)_2Nsk2{dc8GA+ug++@tS@u6zTIVL)vZm+c z{A}D;t?S*~)z)m*$WqWtsuox13B6O+z9SpWvv$7yVH_v% z0Y1cM_yS+zYkY%q_yrg6D=y+PuHX-;Tv{S6la@;>q*|#?YLM1S+oc`SUa4D3Oa0P- zh(Ib9-NDI`Nk@pEW;dat3w9G}eEOn`?AW>MUbXFpFS_sN@iM!$a>2qy(dDaa8#ip; zdKHWUyG6KOTLwPzC)JCOd>8#nOKlD>VR?dUSKqEiv{_Ute(rV460ns#Z#MDP&u8Vp+g6w~!lNW%56b z_ix!*cAi~e7uhAuMj13junKpgk?_48I|<`$XeW#((Srl%g@JyUc!&`1p@73Uf}?m0 zkK;)^h2w;pTH0J5kF1AbNggGv2qt{XFau!*{!#`|+L~x>CexWVGk({O zQh$&-Z(MIoP~U_GuEN*->p0P?e;Cp{8pz@S9TSu!)c*A!0{+&Q?(qB%&wn=XUpW8& E1mFaK5dZ)H From db81b78437d356c13e9f6550f62667713a2c67a1 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:24:17 +0400 Subject: [PATCH 37/41] Delegate model data imports to contribution plugins The model service imported language-model-csv directly and ran the CSV import itself, so adding any further import format meant changing the model service, and the model service knew about a specific plugin's data format. Follows the approach the config service already uses for its sections. The model service now extracts the source text of each contributed import block and forwards it to the plugin's own language service via sendPluginRequest, then merges the returned instances and links with the hand-authored ones. The CSV parsing and file reading move into service-model-csv behind a new model-import request handler. Adds service-model-common with the request and response contract, mirroring service-config-common. Only parser errors fail the plugin-side parse, since the plugin's service has no metamodel document to link cross-references against; the model service has already validated those on the real document, so plugins read names from $refText. Also drops the hardcoded CSV entries from the model language's syntax keywords. Contribution plugins already declare these through additionalKeywords, which the workbench merges in. --- app/package-lock.json | 20 +- .../service-model-common/package.json | 19 ++ .../service-model-common/src/index.ts | 2 + .../src/modelPluginHandler.ts | 104 ++++++++++ .../src/modelPluginTypes.ts | 139 ++++++++++++++ .../service-model-common/tsconfig.json | 23 +++ app/packages/service-model-csv/package.json | 8 +- .../src/handler/csvImportRequestHandler.ts | 59 ++++++ app/packages/service-model-csv/src/index.ts | 5 + app/packages/service-model-csv/tsconfig.json | 3 +- app/packages/service-model/package.json | 7 +- .../src/handler/modelDataHandler.ts | 179 +++++++++++++----- app/packages/service-model/src/index.ts | 5 +- app/packages/service-model/tsconfig.json | 2 +- app/tsconfig.build.json | 3 + 15 files changed, 520 insertions(+), 58 deletions(-) create mode 100644 app/packages/service-model-common/package.json create mode 100644 app/packages/service-model-common/src/index.ts create mode 100644 app/packages/service-model-common/src/modelPluginHandler.ts create mode 100644 app/packages/service-model-common/src/modelPluginTypes.ts create mode 100644 app/packages/service-model-common/tsconfig.json create mode 100644 app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts diff --git a/app/package-lock.json b/app/package-lock.json index a50c7d79..7aee3bbe 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1720,6 +1720,10 @@ "resolved": "packages/service-model", "link": true }, + "node_modules/@mdeo/service-model-common": { + "resolved": "packages/service-model-common", + "link": true + }, "node_modules/@mdeo/service-model-csv": { "resolved": "packages/service-model-csv", "link": true @@ -8185,10 +8189,11 @@ "@mdeo/editor-model": "^0.1.0", "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", - "@mdeo/language-model-csv": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", "lucide": "^0.575.0" }, "devDependencies": { @@ -8197,6 +8202,18 @@ "vite": "^7.3.1" } }, + "packages/service-model-common": { + "name": "@mdeo/service-model-common", + "version": "0.1.0", + "dependencies": { + "@mdeo/language-model": "^0.1.0", + "@mdeo/service-common": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } + }, "packages/service-model-csv": { "name": "@mdeo/service-model-csv", "version": "0.1.0", @@ -8207,6 +8224,7 @@ "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", "lucide": "^0.575.0" }, "devDependencies": { diff --git a/app/packages/service-model-common/package.json b/app/packages/service-model-common/package.json new file mode 100644 index 00000000..4aa92fa2 --- /dev/null +++ b/app/packages/service-model-common/package.json @@ -0,0 +1,19 @@ +{ + "name": "@mdeo/service-model-common", + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist", + "src" + ], + "dependencies": { + "@mdeo/language-model": "^0.1.0", + "@mdeo/service-common": "^0.1.0" + }, + "devDependencies": { + "@types/node": "^25.3.1", + "langium": "^4.2.1" + } +} diff --git a/app/packages/service-model-common/src/index.ts b/app/packages/service-model-common/src/index.ts new file mode 100644 index 00000000..db0f69d5 --- /dev/null +++ b/app/packages/service-model-common/src/index.ts @@ -0,0 +1,2 @@ +export * from "./modelPluginHandler.js"; +export * from "./modelPluginTypes.js"; diff --git a/app/packages/service-model-common/src/modelPluginHandler.ts b/app/packages/service-model-common/src/modelPluginHandler.ts new file mode 100644 index 00000000..cbe6c655 --- /dev/null +++ b/app/packages/service-model-common/src/modelPluginHandler.ts @@ -0,0 +1,104 @@ +import { TextDocument } from "langium"; +import type { AstNode, LangiumDocument } from "langium"; +import { hasParserErrors } from "@mdeo/service-common"; +import type { RequestContext } from "@mdeo/service-common"; +import type { ModelPluginRequestBody } from "./modelPluginTypes.js"; + +/** + * A parsed import block belonging to the requesting plugin. + */ +export interface ModelPluginImport { + /** + * The wrapper node's `$type`, i.e. the wrapper interface name for the import keyword. + */ + type: string; + + /** + * The import block's content node, as defined by the plugin's own grammar. + */ + content: AstNode; +} + +/** + * Discriminated union returned by {@link buildModelPluginDocument}. + * + * - `"empty"` - the request carried no text, or the parsed document contained no + * imports; callers should return an empty but valid response. + * - `"error"` - the text could not be parsed; callers should return `data: null` + * while still reporting tracked dependencies. + * - `"success"` - the document parsed; callers can walk {@link imports}. + */ +export type ModelPluginDocumentResult = + | { type: "empty" } + | { type: "error" } + | { + type: "success"; + /** + * The built Langium document. + */ + document: LangiumDocument; + /** + * The import blocks contributed by this plugin. + */ + imports: ModelPluginImport[]; + /** + * The decoded request body supplied by the model service. + */ + requestBody: ModelPluginRequestBody; + }; + +/** + * Builds a partial model document for an import contribution plugin. + * + * Mirrors `buildConfigPluginDocument`, with one deliberate difference: only + * parser errors are treated as failures, not validation errors. A plugin's + * service has the plugin's own grammar but not the metamodel document, so + * cross-references into the metamodel cannot be linked here and would otherwise + * always report as errors. That costs nothing, because the model service has + * already validated the real `.m` document (including those references) before + * forwarding anything; plugins read referenced names from `$refText`. + * + * The synthetic document is given the originating `.m` file's URI so that paths + * written inside the import block resolve relative to the model file, exactly as + * they do in the model service. + * + * @param context The request context injected by the service framework + * @param languageKey The language ID of this contribution plugin's language + * @returns A discriminated {@link ModelPluginDocumentResult} + */ +export async function buildModelPluginDocument( + context: RequestContext, + languageKey: string +): Promise { + const requestBody = context.body as ModelPluginRequestBody; + const text = requestBody?.text ?? ""; + + if (!text.trim()) { + return { type: "empty" }; + } + + const textDocument = TextDocument.create(requestBody.modelFileUri, languageKey, 0, text); + context.services.shared.workspace.TextDocuments.set(textDocument); + const document = context.services.shared.workspace.LangiumDocumentFactory.fromTextDocument(textDocument); + + await context.services.shared.workspace.DocumentBuilder.build([document], { validation: false }); + if (hasParserErrors(document)) { + return { type: "error" }; + } + + const root = document.parseResult?.value as { imports?: AstNode[] } | undefined; + const imports = root?.imports; + if (root == undefined || !Array.isArray(imports) || imports.length === 0) { + return { type: "empty" }; + } + + return { + type: "success", + document, + imports: imports.map((wrapper) => ({ + type: wrapper.$type, + content: (wrapper as AstNode & { content: AstNode }).content + })), + requestBody + }; +} diff --git a/app/packages/service-model-common/src/modelPluginTypes.ts b/app/packages/service-model-common/src/modelPluginTypes.ts new file mode 100644 index 00000000..df87450d --- /dev/null +++ b/app/packages/service-model-common/src/modelPluginTypes.ts @@ -0,0 +1,139 @@ +import type { ModelDataInstance, ModelDataLink } from "@mdeo/language-model"; +import type { FileDependency, DataDependency } from "@mdeo/service-common"; + +/** + * Description of a single metamodel property, as sent to import plugins. + * + * Import plugins map their own source data (CSV columns, spreadsheet fields, and + * so on) onto these, so they need to know each property's name and type without + * having to resolve the metamodel document themselves. + */ +export interface MetamodelPropertyInfo { + /** + * Name of the property as declared in the metamodel. + */ + name: string; + + /** + * The property's declared type. `reference` means the property is an + * association end rather than an attribute. + */ + type: "string" | "int" | "long" | "double" | "float" | "boolean" | "enum" | "reference"; + + /** + * For enum-typed properties, the names of the enum's entries. + */ + enumEntries?: string[]; + + /** + * True when the property is an association end. + */ + isReference?: boolean; + + /** + * For reference properties, the name of the class being referenced. + */ + referencedClass?: string; +} + +/** + * Description of a single metamodel class, as sent to import plugins. + */ +export interface MetamodelClassInfo { + /** + * Name of the class as declared in the metamodel. + */ + name: string; + + /** + * The class' properties. + */ + properties: MetamodelPropertyInfo[]; +} + +/** + * Request body sent to a model import contribution plugin's request handler. + * + * Mirrors `ConfigPluginRequestBody`: the model service extracts the text of the + * plugin's own import block and hands it back to the plugin's service, so the + * plugin parses and interprets its own syntax rather than the model service + * knowing anything about it. + */ +export interface ModelPluginRequestBody { + /** + * The source text of this plugin's import block content, without the + * surrounding `import ` keywords. + */ + text: string; + + /** + * URI string of the originating `.m` file. Used to build a stable synthetic + * document URI and to resolve paths referenced by the import relative to it. + */ + modelFileUri: string; + + /** + * Path of the metamodel the model imports, as written in the `.m` file. + */ + metamodelPath: string; + + /** + * The metamodel's classes, so the plugin can map its source data onto them + * without resolving the metamodel document itself. + */ + metamodelClasses: MetamodelClassInfo[]; +} + +/** + * The instances and links produced by one import contribution plugin. + */ +export interface ModelPluginData { + /** + * Object instances derived from the plugin's imported data. + */ + instances: ModelDataInstance[]; + + /** + * Links derived from the plugin's imported data. + */ + links: ModelDataLink[]; + + /** + * Non-fatal problems encountered while importing (unknown columns, unresolved + * references, and similar). Present so the host service can surface them. + */ + warnings?: string[]; +} + +/** + * Response returned by a model import contribution plugin's request handler. + * + * As with the config equivalent, dependencies MUST be reported even when + * {@link data} is `null`, so the host service can propagate them to the cache + * and avoid incorrect cache invalidation. + * + * @template T The type of the payload. + */ +export interface ModelPluginRequestResponse { + /** + * The computed import data, or `null` when the plugin could not produce a + * valid result (for example a parse or validation error). + */ + data: T | null; + + /** + * Files (and their versions) read while processing the import. + */ + fileDependencies: FileDependency[]; + + /** + * Other file-data computations consulted while processing the import. + */ + dataDependencies: DataDependency[]; +} + +/** + * Request-handler key used by model import contribution plugin services to + * register and handle import-data requests from the model service. + */ +export const MODEL_PLUGIN_REQUEST_KEY = "model-import"; diff --git a/app/packages/service-model-common/tsconfig.json b/app/packages/service-model-common/tsconfig.json new file mode 100644 index 00000000..c127f254 --- /dev/null +++ b/app/packages/service-model-common/tsconfig.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "references": [ + { + "path": "../plugin" + }, + { + "path": "../language-common" + }, + { + "path": "../language-model" + }, + { + "path": "../service-common" + } + ], + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/app/packages/service-model-csv/package.json b/app/packages/service-model-csv/package.json index 4a707fd1..9b273838 100644 --- a/app/packages/service-model-csv/package.json +++ b/app/packages/service-model-csv/package.json @@ -5,7 +5,10 @@ "type": "module", "main": "dist/index.js", "types": "dist/index.d.ts", - "files": ["dist", "src"], + "files": [ + "dist", + "src" + ], "scripts": { "build:static": "vite build", "watch:static": "vite build --watch", @@ -17,8 +20,9 @@ "@mdeo/language-model": "^0.1.0", "@mdeo/language-model-csv": "^0.1.0", "@mdeo/language-shared": "^0.1.0", - "@mdeo/service-common": "^0.1.0", "@mdeo/plugin": "^0.1.0", + "@mdeo/service-common": "^0.1.0", + "@mdeo/service-model-common": "^0.1.0", "lucide": "^0.575.0" }, "devDependencies": { diff --git a/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts b/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts new file mode 100644 index 00000000..e5ac3f88 --- /dev/null +++ b/app/packages/service-model-csv/src/handler/csvImportRequestHandler.ts @@ -0,0 +1,59 @@ +import type { RequestHandler } from "@mdeo/service-common"; +import { resolveRelativePath } from "@mdeo/language-shared"; +import { + buildModelPluginDocument, + type ModelPluginData, + type ModelPluginRequestResponse +} from "@mdeo/service-model-common"; +import { importCsvEntries, MODEL_CSV_LANGUAGE_KEY } from "@mdeo/language-model-csv"; +import type { CsvImportBlockType, ModelCsvServices } from "@mdeo/language-model-csv"; + +/** + * Handles import-data requests forwarded by the model service. + * + * The model service does not know anything about CSV: it hands over the text of + * the `import CSV { ... }` block together with a description of the metamodel's + * classes, and this handler parses that text with the CSV grammar, reads the + * referenced CSV files, and returns the resulting instances and links. + * + * Class names are read from `$refText` rather than a resolved reference, because + * the metamodel document is not loaded in this service. The model service has + * already validated those references against the real metamodel before + * forwarding, so the names are known to be valid here. + */ +export const csvImportRequestHandler: RequestHandler = async ( + context +): Promise => { + const result = await buildModelPluginDocument(context, MODEL_CSV_LANGUAGE_KEY); + + if (result.type === "empty") { + return { data: { instances: [], links: [] }, ...context.serverApi.getTrackedRequests() }; + } + if (result.type === "error") { + return { data: null, ...context.serverApi.getTrackedRequests() }; + } + + const { document, imports, requestBody } = result; + + const entries = []; + for (const importBlock of imports) { + const content = importBlock.content as CsvImportBlockType; + for (const entry of content.imports ?? []) { + const file = entry.file ?? ""; + if (file === "") { + continue; + } + const csvPath = resolveRelativePath(document, file).path; + const { content: csvText } = await context.serverApi.readFile(csvPath); + entries.push({ + className: entry.class?.$refText ?? "", + csvText + }); + } + } + + const { instances, links, warnings } = importCsvEntries(entries, requestBody.metamodelClasses); + const data: ModelPluginData = { instances, links, warnings }; + + return { data, ...context.serverApi.getTrackedRequests() }; +}; diff --git a/app/packages/service-model-csv/src/index.ts b/app/packages/service-model-csv/src/index.ts index f7aa15ac..4d3f99b6 100644 --- a/app/packages/service-model-csv/src/index.ts +++ b/app/packages/service-model-csv/src/index.ts @@ -11,6 +11,7 @@ import { AST_HANDLER_KEY, startLanguageService } from "@mdeo/service-common"; +import { MODEL_PLUGIN_REQUEST_KEY } from "@mdeo/service-model-common"; import type { LanguagePlugin } from "@mdeo/plugin"; const icon = convertIcon(Table); @@ -36,6 +37,7 @@ const modelCsvLanguagePlugin: LanguagePlugin = { initializePluginContext(); const { modelCsvPluginProvider, createModelCsvContributionPlugin } = await import("@mdeo/language-model-csv"); +const { csvImportRequestHandler } = await import("./handler/csvImportRequestHandler.js"); /** * Plugin definition for the model-csv service. @@ -66,6 +68,9 @@ const modelCsvLanguageConfig: LanguageServiceConfig = { languagePluginProvider: modelCsvPluginProvider, fileDataHandlers: { [AST_HANDLER_KEY]: astHandler + }, + requestHandlers: { + [MODEL_PLUGIN_REQUEST_KEY]: csvImportRequestHandler } }; diff --git a/app/packages/service-model-csv/tsconfig.json b/app/packages/service-model-csv/tsconfig.json index 44840f38..d169b31b 100644 --- a/app/packages/service-model-csv/tsconfig.json +++ b/app/packages/service-model-csv/tsconfig.json @@ -10,7 +10,8 @@ { "path": "../language-model" }, { "path": "../language-model-csv" }, { "path": "../language-shared" }, - { "path": "../service-common" } + { "path": "../service-common" }, + { "path": "../service-model-common" } ], "include": ["src/**/*.ts"], "exclude": ["dist", "node_modules"] diff --git a/app/packages/service-model/package.json b/app/packages/service-model/package.json index 6b0a261e..95210e02 100644 --- a/app/packages/service-model/package.json +++ b/app/packages/service-model/package.json @@ -17,15 +17,16 @@ "@mdeo/editor-model": "^0.1.0", "@mdeo/language-common": "^0.1.0", "@mdeo/language-model": "^0.1.0", + "@mdeo/language-shared": "^0.1.0", "@mdeo/plugin": "^0.1.0", "@mdeo/protocol-model": "^0.1.0", "@mdeo/service-common": "^0.1.0", - "lucide": "^0.575.0", - "@mdeo/language-model-csv": "^0.1.0" + "@mdeo/service-model-common": "^0.1.0", + "lucide": "^0.575.0" }, "devDependencies": { "@types/node": "^25.3.1", "tsx": "^4.21.0", "vite": "^7.3.1" } -} \ No newline at end of file +} diff --git a/app/packages/service-model/src/handler/modelDataHandler.ts b/app/packages/service-model/src/handler/modelDataHandler.ts index ed5316aa..b131bdea 100644 --- a/app/packages/service-model/src/handler/modelDataHandler.ts +++ b/app/packages/service-model/src/handler/modelDataHandler.ts @@ -1,17 +1,76 @@ -import { type ModelData, type ModelServices, ModelDataConverter, Model, getWrapperInterfaceName } from "@mdeo/language-model"; import { - type MetamodelClassInfo, - type MetamodelPropertyInfo, - type CsvImportBlockType, - importCsvEntries -} from "@mdeo/language-model-csv"; -import { hasErrors, type FileDataHandler } from "@mdeo/service-common"; + type ModelData, + type ModelServices, + ModelDataConverter, + Model, + ModelContributionPlugin, + getWrapperInterfaceName +} from "@mdeo/language-model"; +import type { + MetamodelClassInfo, + MetamodelPropertyInfo, + ModelPluginData, + ModelPluginRequestBody, + ModelPluginRequestResponse +} from "@mdeo/service-model-common"; +import { MODEL_PLUGIN_REQUEST_KEY } from "@mdeo/service-model-common"; +import { hasErrors, type FileDataHandler, type FileDependency, type DataDependency } from "@mdeo/service-common"; import { resolveRelativePath } from "@mdeo/language-shared"; +import type { AstNode } from "langium"; export const MODEL_DATA_HANDLER_KEY = "model-data"; +/** + * Extracts a flat description of a metamodel's classes for import plugins. + * + * Import plugins map their own source data onto the metamodel but have no access + * to the metamodel document, so this is sent to them as plain data. + * + * @param metamodelAst The parsed metamodel document root + * @returns One entry per class declared in the metamodel + */ +function extractMetamodelClasses(metamodelAst: { elements?: unknown[] } | undefined): MetamodelClassInfo[] { + return ((metamodelAst?.elements ?? []) as any[]) + .filter((element) => element.$type === "Class") + .map( + (cls): MetamodelClassInfo => ({ + name: cls.name, + properties: (cls.properties ?? []).map((prop: any): MetamodelPropertyInfo => { + const primitiveType = prop.type?.name; + const enumRef = prop.type?.enum?.ref; + const isReference = prop.$type === "AssociationEnd"; + return { + name: prop.name, + type: isReference + ? "reference" + : enumRef + ? "enum" + : ((primitiveType ?? "string") as MetamodelPropertyInfo["type"]), + enumEntries: enumRef ? enumRef.entries?.map((entry: any) => entry.name) : undefined, + isReference, + referencedClass: isReference ? prop.class?.ref?.name : undefined + }; + }) + }) + ); +} + +/** + * File-data handler that computes the structured data for a model file. + * + * Hand-authored objects and links are converted directly. Imported data is not: + * the model service has no knowledge of any particular import format, so for + * each contribution plugin whose import appears in the file it forwards that + * import block's own source text to the plugin's language service and merges + * back the instances and links the plugin returns. This mirrors how the config + * service delegates each section to the plugin that contributes it, and keeps + * the model service independent of CSV or any other future import format. + * + * Returns `null` when the document has errors, or when any plugin reports that + * it could not produce a valid result. + */ export const modelDataHandler: FileDataHandler = async (context) => { - const { instance, fileInfo, serverApi } = context; + const { instance, fileInfo, serverApi, contributionPlugins } = context; if (fileInfo == undefined) { return { @@ -36,14 +95,15 @@ export const modelDataHandler: FileDataHandler throw new Error("Document root is not a Model"); } - const csvImport = model.imports?.find((imp) => (imp as { $type?: string }).$type === getWrapperInterfaceName("CSV")); - const csvImportContent = (csvImport as { content?: CsvImportBlockType } | undefined)?.content; + const converter = new ModelDataConverter(reflection); + const handAuthored = converter.convertModel(model); + + const modelImports = (model.imports ?? []) as (AstNode & { content?: AstNode })[]; + const activeModelPlugins = contributionPlugins.filter(ModelContributionPlugin.is); - if (csvImportContent == undefined) { - const converter = new ModelDataConverter(reflection); - const modelData = converter.convertModel(model); + if (modelImports.length === 0 || activeModelPlugins.length === 0) { return { - data: modelData, + data: handAuthored, ...serverApi.getTrackedRequests() }; } @@ -59,51 +119,72 @@ export const modelDataHandler: FileDataHandler return { data: null, ...serverApi.getTrackedRequests() }; } - const metamodelAst = metamodelDoc.parseResult?.value as any; - const metamodelClasses: MetamodelClassInfo[] = (metamodelAst?.elements ?? []) - .filter((el: any) => el.$type === "Class") - .map((cls: any): MetamodelClassInfo => ({ - name: cls.name, - properties: (cls.properties ?? []).map((prop: any): MetamodelPropertyInfo => { - const primitiveType = prop.type?.name; - const enumRef = prop.type?.enum?.ref; - const isReference = prop.$type === "AssociationEnd"; - return { - name: prop.name, - type: isReference ? "reference" - : enumRef ? "enum" - : (primitiveType ?? "string") as MetamodelPropertyInfo["type"], - enumEntries: enumRef ? enumRef.entries?.map((e: any) => e.name) : undefined, - isReference, - referencedClass: isReference ? prop.class?.ref?.name : undefined - }; - }) - })); + const metamodelClasses = extractMetamodelClasses(metamodelDoc.parseResult?.value as { elements?: unknown[] }); + + const importedInstances: ModelData["instances"] = []; + const importedLinks: ModelData["links"] = []; + const accumulatedFileDeps: FileDependency[] = []; + const accumulatedDataDeps: DataDependency[] = []; + + for (const plugin of activeModelPlugins) { + const partialBlocks: string[] = []; + for (const contributedImport of plugin.imports) { + const wrapperType = getWrapperInterfaceName(contributedImport.name); + for (const modelImport of modelImports) { + if (modelImport.$type !== wrapperType) { + continue; + } + const contentText = modelImport.content?.$cstNode?.text ?? ""; + if (contentText !== "") { + partialBlocks.push(`import ${contributedImport.name} ${contentText}`); + } + } + } + + if (partialBlocks.length === 0) { + continue; + } + + const requestBody: ModelPluginRequestBody = { + text: `using "${metamodelPath}"\n\n${partialBlocks.join("\n")}`, + modelFileUri: fileInfo.uri.toString(), + metamodelPath, + metamodelClasses + }; - const csvEntries = await Promise.all( - csvImportContent.imports.map(async (entry: any) => { - const csvPath: string = resolveRelativePath(document, entry.file).path; - const { content } = await serverApi.readFile(csvPath); + const rawPluginResult = await serverApi.sendPluginRequest( + plugin.languageKey, + MODEL_PLUGIN_REQUEST_KEY, + requestBody + ); + const pluginResponse = rawPluginResult as ModelPluginRequestResponse; + + accumulatedFileDeps.push(...(pluginResponse?.fileDependencies ?? [])); + accumulatedDataDeps.push(...(pluginResponse?.dataDependencies ?? [])); + + if (pluginResponse?.data == null) { + const trackedRequests = serverApi.getTrackedRequests(); return { - className: entry.class?.ref?.name ?? "", - csvText: content + data: null, + fileDependencies: [...trackedRequests.fileDependencies, ...accumulatedFileDeps], + dataDependencies: [...trackedRequests.dataDependencies, ...accumulatedDataDeps] }; - }) - ); - - const result = importCsvEntries(csvEntries, metamodelClasses); + } - const converter = new ModelDataConverter(reflection); - const handAuthored = converter.convertModel(model); + importedInstances.push(...(pluginResponse.data.instances ?? [])); + importedLinks.push(...(pluginResponse.data.links ?? [])); + } const modelData: ModelData = { metamodelPath, - instances: [...handAuthored.instances, ...result.instances], - links: [...handAuthored.links, ...result.links] + instances: [...handAuthored.instances, ...importedInstances], + links: [...handAuthored.links, ...importedLinks] }; + const trackedRequests = serverApi.getTrackedRequests(); return { data: modelData, - ...serverApi.getTrackedRequests() + fileDependencies: [...trackedRequests.fileDependencies, ...accumulatedFileDeps], + dataDependencies: [...trackedRequests.dataDependencies, ...accumulatedDataDeps] }; }; diff --git a/app/packages/service-model/src/index.ts b/app/packages/service-model/src/index.ts index e0fa39ee..6c4df970 100644 --- a/app/packages/service-model/src/index.ts +++ b/app/packages/service-model/src/index.ts @@ -33,9 +33,12 @@ const modelLanguagePlugin: LanguagePlugin = { }, textualEditorPlugin: { languageConfiguration: defaultLanguageConfiguration, + // Only the base language's own keyword. Keywords for contributed imports + // (e.g. `import CSV from`) come from each contribution plugin's + // additionalKeywords and are merged in by the workbench. monarchTokensProvider: serializeMonarchTokensProvider({ ...defaultMonarchTokenProvider, - keywords: ["using", "import", "CSV", "from"] + keywords: ["using"] }) }, isGenerated: false diff --git a/app/packages/service-model/tsconfig.json b/app/packages/service-model/tsconfig.json index 015502c0..aad7cf23 100644 --- a/app/packages/service-model/tsconfig.json +++ b/app/packages/service-model/tsconfig.json @@ -21,7 +21,7 @@ "path": "../service-common" }, { - "path": "../language-model-csv" + "path": "../service-model-common" } ], "include": [ diff --git a/app/tsconfig.build.json b/app/tsconfig.build.json index 84e38a6c..c0bef162 100644 --- a/app/tsconfig.build.json +++ b/app/tsconfig.build.json @@ -92,6 +92,9 @@ { "path": "packages/service-model" }, + { + "path": "packages/service-model-common" + }, { "path": "packages/service-model-csv" }, From 0653f506fcbe97b82dae94f8f636be069079480b Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:59:08 +0400 Subject: [PATCH 38/41] Use plain nested braces for the CSV column mapping The mapping block was introduced by a "with" keyword on the grounds that reusing braces made the grammar ambiguous. That was wrong. Tested against the composed grammar with the keyword removed: a file mixing a mapped and an unmapped import parses with no errors and imports correctly, and the parser reports no ambiguity. The block can only start with "{", while the enclosing import list continues with an ID or ends with "}", so one token of lookahead is enough. The comparison drawn to the model-transformation grammar was also wrong: the rule there separates two adjacent brace blocks, which is a different situation from a single optional block after a string. Uses plain nested braces, matching the nested blocks in the other DSLs. --- .../src/grammar/csvImportRules.ts | 15 +++++++-------- .../src/grammar/csvImportTypes.ts | 4 ++-- app/packages/service-model-csv/src/index.ts | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/app/packages/language-model-csv/src/grammar/csvImportRules.ts b/app/packages/language-model-csv/src/grammar/csvImportRules.ts index 5d05e790..c474c807 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportRules.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportRules.ts @@ -16,13 +16,12 @@ export const CsvColumnMappingRule = createRule("CsvColumnMappingRule") .as(({ set }) => [set("csvColumn", STRING), "=", set("property", ID)]); /** - * The optional explicit mapping list is introduced by the "with" keyword, - * the same way the model-transformation grammar disambiguates two adjacent - * brace blocks in one rule (e.g. `match { pattern } then { block }`): the - * keyword between them is what tells the parser which block it's entering, - * not the bracket type. Reusing bare curly braces here, with nothing between - * `file` and the mapping list's own "{", caused the parser to misread the - * mapping list's opening brace as the start of a new class import. + * A class import, with an optional explicit column mapping block. + * + * The mapping block uses plain nested braces, like the nested blocks in the + * other DSLs. Nothing needs to separate it from `file`: the block can only + * start with "{", while the enclosing import list continues with an ID or ends + * with "}", so the parser can always tell which one it is from a single token. */ export const CsvClassImportRule = createRule("CsvClassImportRule") .returns(CsvClassImport) @@ -31,7 +30,7 @@ export const CsvClassImportRule = createRule("CsvClassImportRule") "from", set("file", STRING), optional( - group("with", "{", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "}") + group("{", many(or(add("mappings", CsvColumnMappingRule), NEWLINE)), "}") ) ]); diff --git a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts index e14f618a..03bcc104 100644 --- a/app/packages/language-model-csv/src/grammar/csvImportTypes.ts +++ b/app/packages/language-model-csv/src/grammar/csvImportTypes.ts @@ -26,9 +26,9 @@ export type CsvColumnMappingType = ASTType; * Format: * ClassName from "path/to/file.csv" * or, with an explicit mapping: - * ClassName from "path/to/file.csv" [ + * ClassName from "path/to/file.csv" { * "csvColumnName" = modelPropertyName - * ] + * } * When no mapping list is given, columns are matched to properties by name. */ export const CsvClassImport = createInterface("CsvClassImport").attrs({ diff --git a/app/packages/service-model-csv/src/index.ts b/app/packages/service-model-csv/src/index.ts index 512e236e..be4532e3 100644 --- a/app/packages/service-model-csv/src/index.ts +++ b/app/packages/service-model-csv/src/index.ts @@ -51,8 +51,8 @@ const modelCsvServicePlugin: ServicePluginDefinition = { contributionPlugins: [ { languageId: "model", - description: "Provides import CSV { ClassName from \"file.csv\" with { column = property } } syntax for the model language", - additionalKeywords: ["import", "CSV", "from", "with"], + description: "Provides import CSV { ClassName from \"file.csv\" { \"column\" = property } } syntax for the model language", + additionalKeywords: ["import", "CSV", "from"], serverContributionPlugins: [createModelCsvContributionPlugin()] } ] From 342a9a9e577173bbabf3759c98744232387eda1e Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:12:22 +0400 Subject: [PATCH 39/41] Keep CSV instance names unique across imports of the same class Row numbering restarted at zero for each import entry, so importing the same class from two files produced two instances called ClassName_0, two called ClassName_1, and so on. Links refer to instances by name, so a collision would silently attach a link to the wrong object. Each entry is now numbered from the count of rows already imported for its class. The diagram numbers its nodes the same way, so node names still match the imported data. --- .../src/features/csvImport.ts | 25 ++++++++++++++++--- .../diagram-server/modelGModelFactory.ts | 8 +++++- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/app/packages/language-model-csv/src/features/csvImport.ts b/app/packages/language-model-csv/src/features/csvImport.ts index 0a571b00..b714565d 100644 --- a/app/packages/language-model-csv/src/features/csvImport.ts +++ b/app/packages/language-model-csv/src/features/csvImport.ts @@ -29,7 +29,18 @@ export function importCsvEntries( const idMap = new Map(); - for (const entry of entries) { + /** + * Row numbering restarts at zero for every entry, but two entries can import + * the same class from different files, which would then generate the same + * instance names twice. Each entry is numbered from the count of rows already + * imported for its class, so names stay unique across the whole model. Links + * refer to instances by name, so a collision would silently point at the + * wrong object. + */ + const nameOffsets = new Map(); + const rowsPerClass = new Map(); + + for (const [entryIndex, entry] of entries.entries()) { const classInfo = metamodelClasses.find(c => c.name === entry.className); if (!classInfo) { warnings.push(`Class '${entry.className}' not found in metamodel — skipping.`); @@ -52,9 +63,13 @@ export function importCsvEntries( warnings.push(`CSV for '${entry.className}' has unknown columns: ${unknownCols.join(", ")} — they will be ignored.`); } + const nameOffset = rowsPerClass.get(entry.className) ?? 0; + nameOffsets.set(entryIndex, nameOffset); + rowsPerClass.set(entry.className, nameOffset + dataRows.length); + dataRows.forEach((row, rowIndex) => { const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); - const instanceName = `${entry.className}_${rowIndex}`; + const instanceName = `${entry.className}_${nameOffset + rowIndex}`; if (idColIndex >= 0) { const idValue = normalizedRow[idColIndex]; @@ -79,7 +94,7 @@ export function importCsvEntries( }); } - for (const entry of entries) { + for (const [entryIndex, entry] of entries.entries()) { const classInfo = metamodelClasses.find(c => c.name === entry.className); if (!classInfo) continue; @@ -96,9 +111,11 @@ export function importCsvEntries( if (refCols.length === 0) continue; + const nameOffset = nameOffsets.get(entryIndex) ?? 0; + dataRows.forEach((row, rowIndex) => { const normalizedRow = normalizeRow(row, header.length, rowIndex + 2, warnings); - const sourceInstanceName = `${entry.className}_${rowIndex}`; + const sourceInstanceName = `${entry.className}_${nameOffset + rowIndex}`; refCols.forEach(colName => { const prop = classInfo.properties.find(p => p.name === colName)!; diff --git a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts index ac125f82..ed3f641b 100644 --- a/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts +++ b/app/packages/language-model/src/features/diagram-server/modelGModelFactory.ts @@ -513,6 +513,10 @@ export class ModelGModelFactory extends BaseGModelFactory { const validatedMetadata = await this.modelState.getValidatedMetadata(); let nodeIndex = 0; + // Numbered per class rather than per import, so two imports of the same + // class from different files do not produce the same instance names. + // Matches how the import itself names them. + const rowsPerClass = new Map(); for (const entry of csvImportContent.imports) { const classRef = entry.class?.ref as ClassType | undefined; if (classRef == undefined) continue; @@ -524,8 +528,10 @@ export class ModelGModelFactory extends BaseGModelFactory { const classChain = resolveClassChain(classRef, this.reflection); const classHierarchy = classChain.map((c) => c.name); const columns = this.resolveCsvColumns(rows[0], classChain); + const nameOffset = rowsPerClass.get(classRef.name) ?? 0; + rowsPerClass.set(classRef.name, nameOffset + rows.length - 1); rows.slice(1).forEach((row: string[], rowIndex: number) => { - const instanceName = `${classRef.name}_${rowIndex}`; + const instanceName = `${classRef.name}_${nameOffset + rowIndex}`; const nodeId = `csv-node-${nodeIndex++}`; const metadata = validatedMetadata.nodes[nodeId]?.meta ?? {}; const node = GObjectNode.builder() From a724e370c6380c57d083a2a38cef075be35e29f7 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:43:52 +0400 Subject: [PATCH 40/41] Allow empty CSV files and scope metadata state to one validation call An empty .csv could not be parsed, because ANY_TEXT requires at least one character and the rule assigned it unconditionally. The CSV language sets newFileAction, so every file made with "Create New CSV" was in a parse error state until something was typed into it. The assignment is now optional, and the interface attribute with it. The terminal still requires a character, since a terminal that matches the empty string would not advance the lexer. ModelMetadataManager also passed the metadata under validation to extractGraphMetadata through an instance field, because the base signature only carries the source model. It is now restored in a finally block, so a nested or concurrent validation cannot observe another document's metadata. --- .../language-csv/src/grammar/csvRules.ts | 13 +++++++++++-- .../language-csv/src/grammar/csvTypes.ts | 12 ++++++++++-- .../diagram-server/modelMetadataManager.ts | 16 +++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/app/packages/language-csv/src/grammar/csvRules.ts b/app/packages/language-csv/src/grammar/csvRules.ts index f60230bd..c78cb92d 100644 --- a/app/packages/language-csv/src/grammar/csvRules.ts +++ b/app/packages/language-csv/src/grammar/csvRules.ts @@ -1,10 +1,19 @@ -import { createTerminal, createRule, WS } from "@mdeo/language-common"; +import { createTerminal, createRule, optional, WS } from "@mdeo/language-common"; import { CsvFile } from "./csvTypes.js"; +/** + * Matches a whole CSV file as one token. The CSV language does no structural + * parsing of its own, the content is interpreted where it is imported. + * + * The pattern requires at least one character, since a terminal that can match + * the empty string would never advance the lexer. An empty file is handled by + * making the assignment optional in the rule below rather than by allowing an + * empty match here. + */ export const ANY_TEXT = createTerminal("ANY_TEXT").as(/[\s\S]+/); export const CsvFileRule = createRule("CsvFileRule") .returns(CsvFile) - .as(({ set }) => [set("content", ANY_TEXT)]); + .as(({ set }) => [optional(set("content", ANY_TEXT))]); export const CsvTerminals = [WS, ANY_TEXT]; diff --git a/app/packages/language-csv/src/grammar/csvTypes.ts b/app/packages/language-csv/src/grammar/csvTypes.ts index 4f2a38af..5596072c 100644 --- a/app/packages/language-csv/src/grammar/csvTypes.ts +++ b/app/packages/language-csv/src/grammar/csvTypes.ts @@ -1,7 +1,15 @@ -import { createInterface, type ASTType } from "@mdeo/language-common"; +import { createInterface, Optional, type ASTType } from "@mdeo/language-common"; +/** + * A CSV file's contents, held as a single blob of text. + * + * `content` is optional because an empty file is a valid CSV file. The + * workbench offers a "Create New CSV" action, which produces an empty file, so + * requiring content would put every newly created CSV straight into a parse + * error state. + */ export const CsvFile = createInterface("CsvFile").attrs({ - content: String + content: Optional(String) }); export type CsvFileType = ASTType; diff --git a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts index 13847e6d..7019694a 100644 --- a/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts +++ b/app/packages/language-model/src/features/diagram-server/modelMetadataManager.ts @@ -69,6 +69,15 @@ export class ModelMetadataManager extends MetadataManager { return { nodes, edges }; } + /** + * The metadata being validated by the current {@link validateMetadata} call. + * + * CSV node metadata has to be carried forward from the incoming metadata + * rather than derived from the model, but `extractGraphMetadata` only + * receives the source model, so this passes it down. It is scoped to a + * single call: it is restored in a `finally` block, so a nested or + * concurrent validation cannot see another document's metadata. + */ private storedCurrentMetadata: GraphMetadata | undefined; override async validateMetadata( @@ -76,8 +85,13 @@ export class ModelMetadataManager extends MetadataManager { currentMetadata: GraphMetadata, lastValidMetadata: GraphMetadata ): Promise { + const previousMetadata = this.storedCurrentMetadata; this.storedCurrentMetadata = currentMetadata; - return super.validateMetadata(sourceModel, currentMetadata, lastValidMetadata); + try { + return await super.validateMetadata(sourceModel, currentMetadata, lastValidMetadata); + } finally { + this.storedCurrentMetadata = previousMetadata; + } } private extractCsvNodeMetadata(sourceModel: PartialModel, nodes: Record): void { From 0e4ec256213cd031233d4801211cff89ad48b139 Mon Sep 17 00:00:00 2001 From: amirrza777 <80536453+amirrza777@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:07:00 +0400 Subject: [PATCH 41/41] Add csv and model-csv proxy entries to the workbench vite config Running the workbench outside Docker had no route to the two new plugin services, so they were unreachable in local dev even though nginx proxies them in the Docker setup. Vite matches proxy prefixes in declaration order, so model-csv has to come before model, the same way model-transformation already does, otherwise /plugin/model swallows it. Plain csv goes last. Added a comment recording that, since the ordering is not obvious from the list itself. --- app/packages/workbench/vite.config.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/app/packages/workbench/vite.config.ts b/app/packages/workbench/vite.config.ts index 1de70a51..5d4690cf 100644 --- a/app/packages/workbench/vite.config.ts +++ b/app/packages/workbench/vite.config.ts @@ -27,6 +27,10 @@ export default defineConfig({ "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp" }, + // Matched by prefix in declaration order, so a longer path has to come + // before any shorter path it starts with: model-transformation and + // model-csv before model, config-optimization and config-mdeo before + // config, and plain csv last. proxy: { "/plugin/model-transformation": { target: "http://localhost:3003", @@ -36,6 +40,14 @@ export default defineConfig({ rewrite: (path) => path.replace(/^\/plugin\/model-transformation/, ""), configure: addCoopCoepHeaders }, + "/plugin/model-csv": { + target: "http://localhost:3007", + changeOrigin: true, + secure: false, + ws: true, + rewrite: (path) => path.replace(/^\/plugin\/model-csv/, ""), + configure: addCoopCoepHeaders + }, "/plugin/metamodel": { target: "http://localhost:3000", changeOrigin: true, @@ -84,6 +96,14 @@ export default defineConfig({ rewrite: (path) => path.replace(/^\/plugin\/config/, ""), configure: addCoopCoepHeaders }, + "/plugin/csv": { + target: "http://localhost:3008", + changeOrigin: true, + secure: false, + ws: true, + rewrite: (path) => path.replace(/^\/plugin\/csv/, ""), + configure: addCoopCoepHeaders + }, "/api": { target: "http://localhost:8080", changeOrigin: true,