diff --git a/bases/lif/mdr_restapi/import_export_endpoints.py b/bases/lif/mdr_restapi/import_export_endpoints.py index 21bb3002..01792741 100644 --- a/bases/lif/mdr_restapi/import_export_endpoints.py +++ b/bases/lif/mdr_restapi/import_export_endpoints.py @@ -23,6 +23,12 @@ async def export_data_model(data_model_id: int, session: AsyncSession = Depends( return data_model +@router.get("/export/portable/{data_model_id}", response_model=ImportDataModelDTO) +async def export_data_model_portable(data_model_id: int, session: AsyncSession = Depends(get_session)): + """Name-based export ready to POST straight back to /import/ on another install.""" + return await import_export_service.export_datamodel_portable(session=session, id=data_model_id) + + @router.get("/export/multiple/", response_model=List[SingleDataModelExportDTO]) async def export_data_model(ids: List[int] = Query(...), session: AsyncSession = Depends(get_session)): data_model = await import_export_service.export_multiple_datamodel(session=session, ids=ids) diff --git a/components/lif/mdr_services/import_export_service.py b/components/lif/mdr_services/import_export_service.py index 916dbe39..174f482a 100644 --- a/components/lif/mdr_services/import_export_service.py +++ b/components/lif/mdr_services/import_export_service.py @@ -14,16 +14,27 @@ ValueSet, ValueSetValue, ) -from lif.mdr_dto.attribute_dto import CreateAttributeDTO -from lif.mdr_dto.datamodel_constraints_dto import CreateDataModelConstraintsDTO -from lif.mdr_dto.entity_association_dto import CreateEntityAssociationDTO -from lif.mdr_dto.entity_attribute_association_dto import CreateEntityAttributeAssociationDTO -from lif.mdr_dto.entity_dto import CreateEntityDTO +from lif.mdr_dto.attribute_dto import AttributeDTO, CreateAttributeDTO +from lif.mdr_dto.datamodel_constraints_dto import CreateDataModelConstraintsDTO, DataModelConstraintsDTO +from lif.mdr_dto.datamodel_dto import CreateDataModelDTO, DataModelDTO +from lif.mdr_dto.entity_association_dto import CreateEntityAssociationDTO, EntityAssociationDTO +from lif.mdr_dto.entity_attribute_association_dto import ( + CreateEntityAttributeAssociationDTO, + EntityAttributeAssociationDTO, +) +from lif.mdr_dto.entity_dto import CreateEntityDTO, EntityDTO from lif.mdr_dto.import_export_dto import ( CreateCloneDTO, - SingleDataModelExportDTO, - ImportDataModelDTO, DataModelExportDTO, + ImportAttributeDTO, + ImportDataModelConstraintsDTO, + ImportDataModelDTO, + ImportEntityAssociationDTO, + ImportEntityDTO, + ImportValueSetDTO, + ImportValueSetValueDTO, + ImportValueSetWithValuesDTO, + SingleDataModelExportDTO, ValueSetExportDTO, ) from lif.mdr_dto.value_set_values_dto import CreateValuesWithValueSetDTO @@ -113,9 +124,9 @@ async def export_datamodel(session: AsyncSession, id: int): async def get_export_dto(session: AsyncSession, data_model_id: int): - # Get entities + # Get entities (get_list_of_entities_for_data_model has no check_base param) total_entity_count, entity_list = await get_list_of_entities_for_data_model( - session=session, data_model_id=data_model_id, pagination=False, check_base=False + session=session, data_model_id=data_model_id, pagination=False ) # Get Attributes @@ -138,10 +149,8 @@ async def get_export_dto(session: AsyncSession, data_model_id: int): # Getting all the transformation transformations = await get_transformations_by_data_model_id(session=session, data_model_id=data_model_id) - # Getting Entity associations - entity_associations = await get_entity_associations_by_data_model_id( - session=session, data_model_id=data_model_id, check_base=False - ) + # Getting Entity associations (get_entity_associations_by_data_model_id has no check_base param) + entity_associations = await get_entity_associations_by_data_model_id(session=session, data_model_id=data_model_id) total_association, entity_attribute_associations = await get_entity_attribute_associations_by_data_model_id( session=session, data_model_id=data_model_id, pagination=False @@ -187,6 +196,218 @@ async def export_multiple_datamodel(session: AsyncSession, ids: list[int]): return data_model_list +def build_import_data_model_dto( + data_model: DataModelDTO, + entity_list: List[EntityDTO], + attribute_list: List[AttributeDTO], + value_set_list: List[ValueSetExportDTO], + entity_associations: List[EntityAssociationDTO], + entity_attribute_associations: List[EntityAttributeAssociationDTO], + data_model_constraints: List[DataModelConstraintsDTO], +) -> ImportDataModelDTO: + """Convert an ID-based export into the name-based ImportDataModelDTO that import_datamodel consumes. + + Every cross-row reference in the export (EntityId, AttributeId, ValueSetId, ParentEntityId, the + constraint ElementId, …) is a primary key from the *source* database — a "DB artifact" that means + nothing in another install. This rewrites each reference as the referenced row's name, the portable + identity that import_datamodel resolves against the rows it freshly creates. + + Scope: a single data model (SourceSchema and other standalone models). OrgLIF/PartnerLIF extended + models whose elements span a separate base model are not handled here — import_datamodel takes a + single ImportDataModelDTO and the base/extended split needs its own design (follow-up). + """ + entity_name_by_id = {e.Id: e.Name for e in entity_list} + attribute_name_by_id = {a.Id: a.Name for a in attribute_list} + value_set_name_by_id = {vs.ValueSet.Id: vs.ValueSet.Name for vs in value_set_list} + # An attribute's owning entity is expressed only through the entity/attribute association table. + entity_name_by_attribute_id = { + eaa.AttributeId: entity_name_by_id.get(eaa.EntityId) for eaa in entity_attribute_associations + } + + import_entities = [ + ImportEntityDTO( + Name=e.Name, + UniqueName=e.UniqueName, + Description=e.Description, + UseConsiderations=e.UseConsiderations, + Required=e.Required, + Array=e.Array, + SourceModel=e.SourceModel, + Notes=e.Notes, + CreationDate=e.CreationDate, + ActivationDate=e.ActivationDate, + DeprecationDate=e.DeprecationDate, + Contributor=e.Contributor, + ContributorOrganization=e.ContributorOrganization, + Extension=e.Extension, + ExtensionNotes=e.ExtensionNotes, + Tags=e.Tags, + ) + for e in entity_list + ] + + import_attributes = [ + ImportAttributeDTO( + Name=a.Name, + DataType=a.DataType or "string", + EntityName=entity_name_by_attribute_id.get(a.Id), + UniqueName=a.UniqueName, + Description=a.Description, + UseConsiderations=a.UseConsiderations, + ValueSetName=value_set_name_by_id.get(a.ValueSetId) if a.ValueSetId else None, + Required=a.Required, + Array=a.Array, + SourceModel=a.SourceModel, + Notes=a.Notes, + CreationDate=a.CreationDate, + ActivationDate=a.ActivationDate, + DeprecationDate=a.DeprecationDate, + Contributor=a.Contributor, + ContributorOrganization=a.ContributorOrganization, + Extension=a.Extension, + ExtensionNotes=a.ExtensionNotes, + ) + for a in attribute_list + ] + + import_value_sets = [ + ImportValueSetWithValuesDTO( + ValueSet=ImportValueSetDTO( + Name=vs.ValueSet.Name, + Description=vs.ValueSet.Description, + UseConsiderations=vs.ValueSet.UseConsiderations, + Notes=vs.ValueSet.Notes, + CreationDate=vs.ValueSet.CreationDate, + ActivationDate=vs.ValueSet.ActivationDate, + DeprecationDate=vs.ValueSet.DeprecationDate, + Contributor=vs.ValueSet.Contributor, + ContributorOrganization=vs.ValueSet.ContributorOrganization, + Extension=vs.ValueSet.Extension, + ExtensionNotes=vs.ValueSet.ExtensionNotes, + ), + Values=[ + ImportValueSetValueDTO( + Value=v.Value, + ValueName=v.ValueName, + Description=v.Description, + UseConsiderations=v.UseConsiderations, + Source=v.Source, + Notes=v.Notes, + CreationDate=v.CreationDate, + ActivationDate=v.ActivationDate, + DeprecationDate=v.DeprecationDate, + Contributor=v.Contributor, + ContributorOrganization=v.ContributorOrganization, + Extension=v.Extension, + ExtensionNotes=v.ExtensionNotes, + # OriginalValueId is intentionally dropped — it is a source-DB value id (artifact). + ) + for v in vs.Values + ], + ) + for vs in value_set_list + ] + + import_entity_associations: List[ImportEntityAssociationDTO] = [] + for ea in entity_associations: + parent_name = entity_name_by_id.get(ea.ParentEntityId) + child_name = entity_name_by_id.get(ea.ChildEntityId) + if not parent_name or not child_name: + logger.warning(f"Skipping entity association {ea.Id}: unresolved parent/child entity name") + continue + import_entity_associations.append( + ImportEntityAssociationDTO( + ParentEntityName=parent_name, + ChildEntityName=child_name, + Relationship=ea.Relationship, + Placement=ea.Placement, + Notes=ea.Notes, + CreationDate=ea.CreationDate, + ActivationDate=ea.ActivationDate, + DeprecationDate=ea.DeprecationDate, + Contributor=ea.Contributor, + ContributorOrganization=ea.ContributorOrganization, + ) + ) + + element_name_by_type = { + DatamodelElementType.Entity: entity_name_by_id, + DatamodelElementType.Attribute: attribute_name_by_id, + DatamodelElementType.ValueSet: value_set_name_by_id, + } + import_constraints: List[ImportDataModelConstraintsDTO] = [] + for c in data_model_constraints: + element_name = element_name_by_type.get(c.ElementType, {}).get(c.ElementId) + if not element_name: + logger.warning(f"Skipping constraint {c.Id}: unresolved element for {c.ElementType}/{c.ElementId}") + continue + import_constraints.append( + ImportDataModelConstraintsDTO( + Name=c.Name, + Description=c.Description, + ForDataModelId=c.ForDataModelId, # source-DB id; import_datamodel remaps it to the new model + ElementType=c.ElementType, + ElementName=element_name, + ConstraintType=c.ConstraintType, + Notes=c.Notes, + CreationDate=c.CreationDate, + ActivationDate=c.ActivationDate, + DeprecationDate=c.DeprecationDate, + Contributor=c.Contributor or "", + ContributorOrganization=c.ContributorOrganization or "", + Deleted=c.Deleted, + ) + ) + + import_data_model = CreateDataModelDTO( + Name=data_model.Name, + Description=data_model.Description, + UseConsiderations=data_model.UseConsiderations, + Type=data_model.Type, + BaseDataModelId=None, # source-DB id; intentionally dropped for portability + Notes=data_model.Notes, + DataModelVersion=data_model.DataModelVersion or "", + ActivationDate=data_model.ActivationDate, + DeprecationDate=data_model.DeprecationDate, + Contributor=data_model.Contributor, + ContributorOrganization=data_model.ContributorOrganization, + State=data_model.State, + Tags=data_model.Tags, + ) + + return ImportDataModelDTO( + DataModel=import_data_model, + Entities=import_entities, + Attributes=import_attributes, + ValueSets=import_value_sets, + EntityAssociation=import_entity_associations, + DataModelConstraints=import_constraints, + ) + + +async def export_datamodel_portable(session: AsyncSession, id: int) -> ImportDataModelDTO: + """Export a data model as a portable, name-based ImportDataModelDTO ready to POST back to /import/.""" + data_model = await get_datamodel_by_id(session=session, id=id) + ( + entity_list, + attribute_list, + value_set_list, + _transformations, # transformations are not part of ImportDataModelDTO yet (#771) + entity_associations, + entity_attribute_associations, + data_model_constraints, + ) = await get_export_dto(session=session, data_model_id=data_model.Id) + return build_import_data_model_dto( + data_model=data_model, + entity_list=entity_list, + attribute_list=attribute_list, + value_set_list=value_set_list, + entity_associations=entity_associations, + entity_attribute_associations=entity_attribute_associations, + data_model_constraints=data_model_constraints, + ) + + async def import_datamodel(session: AsyncSession, data: ImportDataModelDTO): entity_name_id = {} value_set_name_id = {} diff --git a/test/bases/lif/mdr_restapi/test_portable_export_roundtrip.py b/test/bases/lif/mdr_restapi/test_portable_export_roundtrip.py new file mode 100644 index 00000000..a56d6793 --- /dev/null +++ b/test/bases/lif/mdr_restapi/test_portable_export_roundtrip.py @@ -0,0 +1,203 @@ +"""End-to-end export->import round-trip for the name-based portable serializer. + +Exercises the whole portability path against a live Postgres DB: + + seed a source model (entities, an attribute + association, an entity reference, + a data-model constraint) + -> export_datamodel_portable (ID-based rows rewritten to name-based refs) + -> import_datamodel (names resolved against freshly created rows) + +and asserts every cross-row reference survives as a *name*: the attribute lands on the +right entity, the entity association points parent->child, and the constraint targets the +right element — none of it leaking the source database's primary keys. +""" + +from sqlmodel import select + +from lif.datatypes.mdr_sql_model import ( + Attribute, + DataModel, + DataModelConstraints, + DataModelType, + DatamodelElementType, + Entity, + EntityAssociation, + EntityAttributeAssociation, + EntityPlacementType, + ValueSet, + ValueSetValue, +) +from lif.mdr_services.import_export_service import export_datamodel_portable, import_datamodel + + +async def _seed_source_model(session): + dm = DataModel( + Name="PortableRoundTripSource", + Type=DataModelType.SourceSchema, + DataModelVersion="1.0", + ContributorOrganization="UniconQA", + Deleted=False, + ) + session.add(dm) + await session.commit() + await session.refresh(dm) + + person = Entity(Name="Person", UniqueName="Person", DataModelId=dm.Id, Array="No", Required="No", Deleted=False) + org = Entity( + Name="Organization", UniqueName="Organization", DataModelId=dm.Id, Array="No", Required="No", Deleted=False + ) + session.add(person) + session.add(org) + await session.commit() + await session.refresh(person) + await session.refresh(org) + + gender_vs = ValueSet(Name="GenderCode", DataModelId=dm.Id, Deleted=False) + session.add(gender_vs) + await session.commit() + await session.refresh(gender_vs) + session.add(ValueSetValue(ValueSetId=gender_vs.Id, DataModelId=dm.Id, Value="F", ValueName="Female", Deleted=False)) + + first_name = Attribute( + Name="firstName", UniqueName="firstName", DataType="string", DataModelId=dm.Id, Deleted=False + ) + gender = Attribute( + Name="gender", UniqueName="gender", DataType="string", DataModelId=dm.Id, ValueSetId=gender_vs.Id, Deleted=False + ) + session.add(first_name) + session.add(gender) + await session.commit() + await session.refresh(first_name) + await session.refresh(gender) + + session.add(EntityAttributeAssociation(EntityId=person.Id, AttributeId=first_name.Id, Deleted=False)) + session.add(EntityAttributeAssociation(EntityId=person.Id, AttributeId=gender.Id, Deleted=False)) + session.add( + EntityAssociation( + ParentEntityId=person.Id, + ChildEntityId=org.Id, + Relationship="employedBy", + Placement=EntityPlacementType.Reference, + Deleted=False, + ) + ) + session.add( + DataModelConstraints( + ForDataModelId=dm.Id, + ElementType=DatamodelElementType.Entity, + ElementId=person.Id, + ConstraintType="Required", + Contributor="tester", + ContributorOrganization="UniconQA", + Deleted=False, + ) + ) + await session.commit() + return dm + + +async def test_portable_export_import_roundtrip(test_db_session): + session = test_db_session + source = await _seed_source_model(session) + + portable = await export_datamodel_portable(session, source.Id) + + # The serializer must have rewritten every DB-id reference as a name. + assert portable.DataModel.BaseDataModelId is None + attr = next(a for a in portable.Attributes if a.Name == "firstName") + assert attr.EntityName == "Person" + gender_attr = next(a for a in portable.Attributes if a.Name == "gender") + assert gender_attr.EntityName == "Person" + assert gender_attr.ValueSetName == "GenderCode" # ValueSetId rewritten to its name + assert any( + ea.ParentEntityName == "Person" and ea.ChildEntityName == "Organization" for ea in portable.EntityAssociation + ) + assert [c.ElementName for c in portable.DataModelConstraints] == ["Person"] + + # Import into a fresh model on the same DB (rename to avoid the unique-name guard). + portable.DataModel.Name = "PortableRoundTripTarget" + result = await import_datamodel(session, portable) + assert result == {"ok": True} + + target = ( + (await session.execute(select(DataModel).where(DataModel.Name == "PortableRoundTripTarget"))).scalars().first() + ) + assert target is not None and target.Id != source.Id + + entities = ( + (await session.execute(select(Entity).where(Entity.DataModelId == target.Id, Entity.Deleted == False))) + .scalars() + .all() + ) + by_name = {e.Name: e.Id for e in entities} + assert sorted(by_name) == ["Organization", "Person"] + + # Attribute resolved onto the right entity via its EntityName. + target_attr = ( + ( + await session.execute( + select(Attribute).where(Attribute.DataModelId == target.Id, Attribute.Name == "firstName") + ) + ) + .scalars() + .first() + ) + assert target_attr is not None + eaa = ( + ( + await session.execute( + select(EntityAttributeAssociation).where( + EntityAttributeAssociation.EntityId == by_name["Person"], + EntityAttributeAssociation.AttributeId == target_attr.Id, + ) + ) + ) + .scalars() + .first() + ) + assert eaa is not None + + # Entity association resolved parent->child with its relationship preserved. + assoc = ( + ( + await session.execute( + select(EntityAssociation) + .join(Entity, Entity.Id == EntityAssociation.ParentEntityId) + .where(Entity.DataModelId == target.Id, EntityAssociation.Deleted == False) + ) + ) + .scalars() + .first() + ) + assert assoc is not None + assert assoc.ParentEntityId == by_name["Person"] + assert assoc.ChildEntityId == by_name["Organization"] + assert assoc.Relationship == "employedBy" + + # Value set recreated and the gender attribute re-linked to it by name. + target_vs = ( + ( + await session.execute( + select(ValueSet).where(ValueSet.DataModelId == target.Id, ValueSet.Name == "GenderCode") + ) + ) + .scalars() + .first() + ) + assert target_vs is not None + target_gender = ( + (await session.execute(select(Attribute).where(Attribute.DataModelId == target.Id, Attribute.Name == "gender"))) + .scalars() + .first() + ) + assert target_gender is not None and target_gender.ValueSetId == target_vs.Id + + # Constraint persisted, element resolved by name, ForDataModelId remapped to the new model. + constraint = ( + (await session.execute(select(DataModelConstraints).where(DataModelConstraints.ForDataModelId == target.Id))) + .scalars() + .first() + ) + assert constraint is not None + assert constraint.ElementType == DatamodelElementType.Entity + assert constraint.ElementId == by_name["Person"]