diff --git a/src/Program.ts b/src/Program.ts index 6f1167f67..7b453bc46 100644 --- a/src/Program.ts +++ b/src/Program.ts @@ -1,7 +1,7 @@ import * as assert from 'assert'; import * as fsExtra from 'fs-extra'; import * as path from 'path'; -import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location, DocumentSymbol, CancellationToken } from 'vscode-languageserver'; +import type { CodeAction, CompletionItem, Position, Range, SignatureInformation, Location, DocumentSymbol, CancellationToken, WorkspaceEdit } from 'vscode-languageserver'; import { CancellationTokenSource, CompletionItemKind } from 'vscode-languageserver'; import type { BsConfig, FinalizedBsConfig } from './BsConfig'; import { Scope } from './Scope'; @@ -24,7 +24,9 @@ import { isBrsFile, isXmlFile, isXmlScope, isNamespaceStatement } from './astUti import type { FunctionStatement, NamespaceStatement } from './parser/Statement'; import { BscPlugin } from './bscPlugin/BscPlugin'; import { AstEditor } from './astUtils/AstEditor'; +import type { SourceNode } from 'source-map'; import type { SourceMapGenerator } from 'source-map'; +import { BrsTranspileState } from './parser/BrsTranspileState'; import type { Statement } from './parser/AstNode'; import { CallExpressionInfo } from './bscPlugin/CallExpressionInfo'; import { SignatureHelpUtil } from './bscPlugin/SignatureHelpUtil'; @@ -57,6 +59,18 @@ export interface SignatureInfoObj { signature: SignatureInformation; } +/** + * Context for a single CDATA block. Returned by `Program.resolveCdataContext` when a + * position falls inside an inline script block. Because synthetic BrsFiles are created + * with offset-padded content, their positions are already in parent XML coordinate space + * — no coordinate transformation is required. + */ +export interface CdataContext { + brsFile: BrsFile; + xmlFile: XmlFile; + cdataRange: Range; +} + export class Program { constructor( /** @@ -158,6 +172,15 @@ export class Program { public files = {} as Record; private pkgMap = {} as Record; + /** + * When set, `getDiagnostics()` keeps this synthetic BrsFile's diagnostics associated with + * the BrsFile rather than remapping them to the parent XmlFile. Set for the duration of any + * plugin event whose `event.file` is a synthetic BrsFile, so that plugins calling + * `program.getDiagnostics()` from inside the handler get results where + * `x.file === event.file` works correctly (e.g. a plugin implementing "fix all"). + */ + private _cdataDiagnosticsContext: BrsFile | undefined; + private scopes = {} as Record; protected addScope(scope: Scope) { @@ -299,10 +322,134 @@ export class Program { }); this.logger.info(`diagnostic counts: total=${chalk.yellow(diagnostics.length.toString())}, after filter=${chalk.yellow(filteredDiagnostics.length.toString())}`); - return filteredDiagnostics; + return this._cdataDiagnosticsContext + ? this.partialRemapSyntheticFileDiagnostics(filteredDiagnostics, this._cdataDiagnosticsContext) + : this.remapSyntheticFileDiagnostics(filteredDiagnostics); + }); + } + + /** + * Redirects diagnostics from synthetic CDATA BrsFiles to their parent XmlFile. + * Because synthetic files are created with offset-padded content, ranges are already + * in XML coordinate space — only the file reference needs updating. + */ + private remapSyntheticFileDiagnostics(diagnostics: BsDiagnostic[]): BsDiagnostic[] { + return diagnostics.map(diagnostic => { + if (!diagnostic.file?.isSynthetic) { + return diagnostic; + } + const parentXmlFile = (diagnostic.file as BrsFile).parentXmlFile; + if (!parentXmlFile) { + return diagnostic; + } + return { ...diagnostic, file: parentXmlFile }; }); } + /** + * Like `remapSyntheticFileDiagnostics`, but keeps `contextFile`'s diagnostics pointing + * at the synthetic BrsFile so plugins can match by file identity during code action events. + */ + private partialRemapSyntheticFileDiagnostics(diagnostics: BsDiagnostic[], contextFile: BrsFile): BsDiagnostic[] { + return diagnostics.map(diagnostic => { + if (!diagnostic.file?.isSynthetic || diagnostic.file === contextFile) { + return diagnostic; + } + const parentXmlFile = (diagnostic.file as BrsFile).parentXmlFile; + if (!parentXmlFile) { + return diagnostic; + } + return { ...diagnostic, file: parentXmlFile }; + }); + } + + /** + * Emit a plugin event with `_cdataDiagnosticsContext` set for the duration if `file` is a + * synthetic BrsFile. This ensures plugins that call `program.getDiagnostics()` from inside + * the handler receive diagnostics still associated with the BrsFile, so that + * `x.file === event.file` identity checks work correctly. + */ + private emitWithSyntheticFileContext(file: BscFile | undefined, emit: () => void) { + if (isBrsFile(file) && file.isSynthetic) { + this._cdataDiagnosticsContext = file; + try { + emit(); + } finally { + this._cdataDiagnosticsContext = undefined; + } + } else { + emit(); + } + } + + /** + * Returns the CDATA metadata and corresponding synthetic BrsFile for the CDATA block whose + * range intersects `range` within `xmlFile`, or `undefined` if no block matches. + */ + private findCdataInfoForRange(xmlFile: XmlFile, range: Range): { meta: { xmlFile: XmlFile; cdataRange: Range }; brsFile: BrsFile } | undefined { + for (const pkgPath of xmlFile.inlineScriptPkgPaths) { + const brsFile = this.getFile(pkgPath); + if (!brsFile?.cdataScript?.cdata) { + continue; + } + if (util.rangesIntersectOrTouch(brsFile.cdataScript.cdata.range, range)) { + return { meta: { xmlFile: xmlFile, cdataRange: brsFile.cdataScript.cdata.range }, brsFile: brsFile }; + } + } + return undefined; + } + + /** + * After code actions are generated using a synthetic BrsFile as the target, this substitutes + * the synthetic file URI with the parent XML file URI in any workspace edit changes. + * Ranges are already in XML coordinate space and need no adjustment. + */ + private remapCodeActionChangesToXml(codeActions: CodeAction[], brsFile: BrsFile, xmlFile: XmlFile) { + const syntheticUri = URI.file(brsFile.srcPath).toString(); + const xmlUri = URI.file(xmlFile.srcPath).toString(); + for (const action of codeActions) { + const changes = (action.edit as WorkspaceEdit)?.changes; + if (!changes?.[syntheticUri]) { + continue; + } + const syntheticEdits = changes[syntheticUri]; + delete changes[syntheticUri]; + if (!changes[xmlUri]) { + changes[xmlUri] = []; + } + changes[xmlUri].push(...syntheticEdits); + } + } + + /** + * Given an XmlFile and a cursor position, returns a `CdataContext` if the position falls + * inside a `` block, or `undefined` if it does not. + * Because synthetic files use offset-padded content, no position transformation is needed — + * pass the original XML-space position directly to the synthetic BrsFile's handlers. + */ + public resolveCdataContext(xmlFile: XmlFile, position: Position): CdataContext | undefined { + const pointRange = util.createRange(position.line, position.character, position.line, position.character); + const info = this.findCdataInfoForRange(xmlFile, pointRange); + if (!info) { + return undefined; + } + return { + brsFile: info.brsFile, + xmlFile: xmlFile, + cdataRange: info.meta.cdataRange + }; + } + + /** + * Substitutes the synthetic BrsFile URI with the parent XML file URI in any Location + * entries that reference it. Ranges are already in XML coordinate space. + */ + private remapLocationsFromSynthetic(locations: Location[], cdataCtx: CdataContext): Location[] { + const syntheticUri = URI.file(cdataCtx.brsFile.srcPath).toString(); + const xmlUri = URI.file(cdataCtx.xmlFile.srcPath).toString(); + return locations.map(loc => (loc.uri === syntheticUri ? { uri: xmlUri, range: loc.range } : loc)); + } + public addDiagnostics(diagnostics: BsDiagnostic[]) { this.diagnostics.push(...diagnostics); } @@ -399,6 +546,16 @@ export class Program { */ public setFile(fileEntry: FileObj, fileContents: string): T; public setFile(fileParam: FileObj | string, fileContents: string): T { + return this.setFileInternal(fileParam, fileContents); + } + + /** + * Internal implementation of setFile that accepts optional BrsFile parse options. + * The extra `parseOptions` parameter is intentionally not exposed on the public overloads — it is + * only used when registering synthetic inline CDATA BrsFiles so that the lexer can start its + * position tracking at the correct XML-space offset without needing a side-channel map. + */ + private setFileInternal(fileParam: FileObj | string, fileContents: string, parseOptions?: { startLine?: number; startCharacter?: number }, configure?: (file: BrsFile) => void): T { //normalize the file paths const { srcPath, pkgPath } = this.getPaths(fileParam, this.options.rootDir); @@ -416,6 +573,12 @@ export class Program { new BrsFile(srcPath, pkgPath, this) ); + // Apply any caller-provided configuration (e.g. marking synthetic files) before + // parsing so that `afterFileParse` fires with the correct state — allowing + // `emitWithSyntheticFileContext` to set `_cdataDiagnosticsContext` for plugins + // that call `getDiagnostics()` from inside the handler. + configure?.(brsFile); + //add file to the `source` dependency list if (brsFile.pkgPath.startsWith(startOfSourcePkgPath)) { this.createSourceScope(); @@ -431,11 +594,11 @@ export class Program { this.plugins.emit('beforeFileParse', sourceObj); this.logger.time(LogLevel.debug, ['parse', chalk.green(srcPath)], () => { - brsFile.parse(sourceObj.source); + brsFile.parse(sourceObj.source, parseOptions); }); //notify plugins that this file has finished parsing - this.plugins.emit('afterFileParse', brsFile); + this.emitWithSyntheticFileContext(brsFile, () => this.plugins.emit('afterFileParse', brsFile)); file = brsFile; @@ -469,6 +632,37 @@ export class Program { file = xmlFile; + //register synthetic BrsFiles for any inline CDATA script blocks. + //these are treated as first-class files so all plugins (linters, validators, etc.) see them normally. + let cdataScriptIndex = 0; + for (const script of xmlFile.ast.component?.scripts ?? []) { + if (script.cdata) { + const inlinePkgPath = xmlFile.inlineScriptPkgPaths[cdataScriptIndex++]; + // Pass the raw CDATA text directly as fileContents. startLine/startCharacter + // tell the lexer to start its position counters at the correct XML-space + // offset, so all token ranges are already in parent XML coordinate space. + // Consumers that need line-indexed text (e.g. SignatureHelpUtil) use the + // parentXmlFile's fileContents instead of the synthetic file's. + const cdataRange = script.cdata.range; + const contentStartChar = cdataRange.start.character + '(inlinePkgPath, rawSource, { + startLine: cdataRange.start.line, + startCharacter: contentStartChar + }, (file) => { + file.isSynthetic = true; + }); + inlineFile.excludeFromOutput = true; + inlineFile.parentXmlFile = xmlFile; + inlineFile.cdataScript = script; + script.cdataTranspile = (state) => { + return inlineFile.needsTranspiled + ? this.transpileSyntheticBrsFileToSourceNode(inlineFile, state.srcPath) + : undefined; + }; + } + } + //create a new scope for this xml file let scope = new XmlScope(xmlFile, this); this.addScope(scope); @@ -729,21 +923,23 @@ export class Program { }) .forEach(() => Object.values(this.files), (file) => { if (!file.isValidated) { - this.plugins.emit('beforeFileValidate', { - program: this, - file: file - }); - - //emit an event to allow plugins to contribute to the file validation process - this.plugins.emit('onFileValidate', { - program: this, - file: file + this.emitWithSyntheticFileContext(file, () => { + this.plugins.emit('beforeFileValidate', { + program: this, + file: file + }); + + //emit an event to allow plugins to contribute to the file validation process + this.plugins.emit('onFileValidate', { + program: this, + file: file + }); + //call file.validate() IF the file has that function defined + file.validate?.(); + file.isValidated = true; + + this.plugins.emit('afterFileValidate', file); }); - //call file.validate() IF the file has that function defined - file.validate?.(); - file.isValidated = true; - - this.plugins.emit('afterFileValidate', file); } }) .forEach(Object.values(this.scopes), (scope) => { @@ -960,15 +1156,17 @@ export class Program { return []; } + const effectiveFile = isXmlFile(file) ? (this.resolveCdataContext(file, position)?.brsFile ?? file) : file; + //find the scopes for this file - let scopes = this.getScopesForFile(file); + let scopes = this.getScopesForFile(effectiveFile); //if there are no scopes, include the global scope so we at least get the built-in functions scopes = scopes.length > 0 ? scopes : [this.globalScope]; const event: ProvideCompletionsEvent = { program: this, - file: file, + file: effectiveFile, scopes: scopes, position: position, completions: [] @@ -1007,9 +1205,12 @@ export class Program { return []; } + const cdataCtx = isXmlFile(file) ? this.resolveCdataContext(file, position) : undefined; + const effectiveFile = cdataCtx?.brsFile ?? file; + const event: ProvideDefinitionEvent = { program: this, - file: file, + file: effectiveFile, position: position, definitions: [] }; @@ -1017,7 +1218,7 @@ export class Program { this.plugins.emit('beforeProvideDefinition', event); this.plugins.emit('provideDefinition', event); this.plugins.emit('afterProvideDefinition', event); - return event.definitions; + return cdataCtx ? this.remapLocationsFromSynthetic(event.definitions, cdataCtx) : event.definitions; } /** @@ -1027,16 +1228,19 @@ export class Program { let file = this.getFile(srcPath); let result: Hover[]; if (file) { + const effectiveFile = isXmlFile(file) ? (this.resolveCdataContext(file, position)?.brsFile ?? file) : file; + const event = { program: this, - file: file, + file: effectiveFile, position: position, - scopes: this.getScopesForFile(file), + scopes: this.getScopesForFile(effectiveFile), hovers: [] } as ProvideHoverEvent; this.plugins.emit('beforeProvideHover', event); this.plugins.emit('provideHover', event); this.plugins.emit('afterProvideHover', event); + result = event.hovers; } @@ -1049,19 +1253,41 @@ export class Program { */ public getDocumentSymbols(srcPath: string): DocumentSymbol[] | undefined { let file = this.getFile(srcPath); - if (file) { - const event: ProvideDocumentSymbolsEvent = { - program: this, - file: file, - documentSymbols: [] - }; - this.plugins.emit('beforeProvideDocumentSymbols', event); - this.plugins.emit('provideDocumentSymbols', event); - this.plugins.emit('afterProvideDocumentSymbols', event); - return event.documentSymbols; - } else { + if (!file) { return undefined; } + const event: ProvideDocumentSymbolsEvent = { + program: this, + file: file, + documentSymbols: [] + }; + this.plugins.emit('beforeProvideDocumentSymbols', event); + this.plugins.emit('provideDocumentSymbols', event); + this.plugins.emit('afterProvideDocumentSymbols', event); + + // For XML files, also collect symbols from each inline CDATA block. + // Ranges are already in XML coordinate space — no remapping needed. + if (isXmlFile(file)) { + for (const pkgPath of file.inlineScriptPkgPaths) { + const brsFile = this.getFile(pkgPath); + if (!brsFile) { + continue; + } + const cdataEvent: ProvideDocumentSymbolsEvent = { + program: this, + file: brsFile, + documentSymbols: [] + }; + this.emitWithSyntheticFileContext(brsFile, () => { + this.plugins.emit('beforeProvideDocumentSymbols', cdataEvent); + this.plugins.emit('provideDocumentSymbols', cdataEvent); + this.plugins.emit('afterProvideDocumentSymbols', cdataEvent); + }); + event.documentSymbols.push(...cdataEvent.documentSymbols); + } + } + + return event.documentSymbols; } /** @@ -1071,24 +1297,38 @@ export class Program { const codeActions = [] as CodeAction[]; const file = this.getFile(srcPath); if (file) { + // resolveCdataContext uses range.start as a probe point; findCdataInfoForRange + // is used internally to check intersection with the full range. + const cdataCtx = isXmlFile(file) ? this.resolveCdataContext(file, range.start) : undefined; + + // When the range falls inside a CDATA block, redirect the event to the synthetic + // BrsFile so that BrsFile-specific code actions work correctly. + const effectiveFile = cdataCtx?.brsFile ?? file; + const diagnostics = this //get all current diagnostics (filtered by diagnostic filters) .getDiagnostics() //only keep diagnostics related to this file - .filter(x => x.file === file) + .filter(x => x.file === effectiveFile) //only keep diagnostics that touch this range .filter(x => util.rangesIntersectOrTouch(x.range, range)); - const scopes = this.getScopesForFile(file); + const scopes = this.getScopesForFile(effectiveFile); - this.plugins.emit('onGetCodeActions', { - program: this, - file: file, - range: range, - diagnostics: diagnostics, - scopes: scopes, - codeActions: codeActions + this.emitWithSyntheticFileContext(effectiveFile, () => { + this.plugins.emit('onGetCodeActions', { + program: this, + file: effectiveFile, + range: range, + diagnostics: diagnostics, + scopes: scopes, + codeActions: codeActions + }); }); + + if (cdataCtx) { + this.remapCodeActionChangesToXml(codeActions, cdataCtx.brsFile, file as XmlFile); + } } return codeActions; } @@ -1098,24 +1338,51 @@ export class Program { */ public getSemanticTokens(srcPath: string): SemanticToken[] | undefined { const file = this.getFile(srcPath); - if (file) { - const result = [] as SemanticToken[]; - this.plugins.emit('onGetSemanticTokens', { - program: this, - file: file, - scopes: this.getScopesForFile(file), - semanticTokens: result - }); - return result; + if (!file) { + return undefined; + } + const result = [] as SemanticToken[]; + this.plugins.emit('onGetSemanticTokens', { + program: this, + file: file, + scopes: this.getScopesForFile(file), + semanticTokens: result + }); + + // For XML files, also collect semantic tokens from each inline CDATA block. + // Ranges are already in XML coordinate space — no remapping needed. + if (isXmlFile(file)) { + for (const pkgPath of file.inlineScriptPkgPaths) { + const brsFile = this.getFile(pkgPath); + if (!brsFile) { + continue; + } + const cdataTokens = [] as SemanticToken[]; + this.emitWithSyntheticFileContext(brsFile, () => { + this.plugins.emit('onGetSemanticTokens', { + program: this, + file: brsFile, + scopes: this.getScopesForFile(brsFile), + semanticTokens: cdataTokens + }); + }); + result.push(...cdataTokens); + } } + + return result; } public getSignatureHelp(filePath: string, position: Position): SignatureInfoObj[] { - let file: BrsFile = this.getFile(filePath); - if (!file || !isBrsFile(file)) { + let file = this.getFile(filePath); + if (!file) { + return []; + } + const effectiveFile = isXmlFile(file) ? (this.resolveCdataContext(file, position)?.brsFile ?? file) : file; + if (!isBrsFile(effectiveFile)) { return []; } - let callExpressionInfo = new CallExpressionInfo(file, position); + let callExpressionInfo = new CallExpressionInfo(effectiveFile, position); let signatureHelpUtil = new SignatureHelpUtil(); return signatureHelpUtil.getSignatureHelpItems(callExpressionInfo); } @@ -1127,9 +1394,12 @@ export class Program { return null; } + const cdataCtx = isXmlFile(file) ? this.resolveCdataContext(file, position) : undefined; + const effectiveFile = cdataCtx?.brsFile ?? file; + const event: ProvideReferencesEvent = { program: this, - file: file, + file: effectiveFile, position: position, references: [] }; @@ -1138,7 +1408,7 @@ export class Program { this.plugins.emit('provideReferences', event); this.plugins.emit('afterProvideReferences', event); - return event.references; + return cdataCtx ? this.remapLocationsFromSynthetic(event.references, cdataCtx) : event.references; } /** @@ -1219,6 +1489,46 @@ export class Program { return Promise.resolve(result); } + /** + * Transpile a synthetic CDATA BrsFile and return a SourceNode suitable for embedding + * back into the parent XML file's transpile output. Plugin beforeFileTranspile events are + * fired so AST edits (e.g. built-in transpile transforms) are applied. afterFileTranspile + * is intentionally skipped — it is designed for standalone file output, not embedded content. + * + * Because synthetic BrsFiles are created with offset-padded content, their token positions + * already align with positions in the parent XML file. Overriding state.srcPath to the XML + * file's path makes the resulting SourceNode reference the XML file directly, producing a + * correct unified source map. + */ + public transpileSyntheticBrsFileToSourceNode(brsFile: BrsFile, xmlSrcPath: string): SourceNode { + const editor = new AstEditor(); + + this.emitWithSyntheticFileContext(brsFile, () => { + this.plugins.emit('beforeFileTranspile', { + program: this, + file: brsFile, + outputPath: undefined, + editor: editor + }); + }); + if (editor.hasChanges) { + editor.setProperty(brsFile, 'needsTranspiled', true); + } + + // Override srcPath so every SourceNode references the XML file at the correct position + const state = new BrsTranspileState(brsFile); + state.srcPath = xmlSrcPath; + + const sourceNode = util.sourceNodeFromTranspileResult( + null, null, state.srcPath, brsFile.ast.transpile(state) + ); + + state.editor.undoAll(); + editor.undoAll(); + + return sourceNode; + } + /** * Internal function used to transpile files. * This does not write anything to the file system @@ -1329,6 +1639,10 @@ export class Program { //mark this file as processed so we don't process it more than once processedFiles.add(outputPath?.toLowerCase()); + if (file.excludeFromOutput) { + return; + } + if (!this.options.pruneEmptyCodeFiles || !file.canBePruned) { //skip transpiling typedef files if (isBrsFile(file) && file.isTypedef) { diff --git a/src/ProgramBuilder.ts b/src/ProgramBuilder.ts index 9e13ccf85..887ed546e 100644 --- a/src/ProgramBuilder.ts +++ b/src/ProgramBuilder.ts @@ -250,6 +250,11 @@ export class ProgramBuilder { throw new Error('Internal invariant exception: somehow file watcher ran before `ProgramBuilder.run()`'); } thePath = s`${path.resolve(this.rootDir, thePath)}`; + //ignore file events for synthetic files (e.g. extracted CDATA scripts) — they are managed + //programmatically by XmlFile and don't exist on disk in the source directory + if (this.program.getFile(thePath)?.isSynthetic) { + return; + } if (event === 'add' || event === 'change') { const fileObj = { src: thePath, diff --git a/src/bscPlugin/SignatureHelpUtil.ts b/src/bscPlugin/SignatureHelpUtil.ts index f0b052d04..df8b2974b 100644 --- a/src/bscPlugin/SignatureHelpUtil.ts +++ b/src/bscPlugin/SignatureHelpUtil.ts @@ -113,7 +113,11 @@ export class SignatureHelpUtil { const documentation = functionComments.join('').trim(); - const lines = util.splitIntoLines(file.fileContents); + // Synthetic BrsFiles (inline CDATA blocks) store only the raw CDATA text, so their + // fileContents cannot be indexed by XML-space line numbers. Use the parent XML file's + // contents instead — it has the correct text at every XML-coordinate line number. + const contentsForLines = file.parentXmlFile?.fileContents ?? file.fileContents; + const lines = util.splitIntoLines(contentsForLines); let key = statement.name.text + documentation; const params = [] as ParameterInformation[]; for (const param of func.parameters) { diff --git a/src/bscPlugin/symbols/WorkspaceSymbolProcessor.ts b/src/bscPlugin/symbols/WorkspaceSymbolProcessor.ts index cc5eb4561..23be8423f 100644 --- a/src/bscPlugin/symbols/WorkspaceSymbolProcessor.ts +++ b/src/bscPlugin/symbols/WorkspaceSymbolProcessor.ts @@ -1,7 +1,9 @@ -import { isBrsFile } from '../../astUtils/reflection'; +import { isBrsFile, isXmlFile } from '../../astUtils/reflection'; import type { BrsFile } from '../../files/BrsFile'; +import type { XmlFile } from '../../files/XmlFile'; import type { ProvideWorkspaceSymbolsEvent } from '../../interfaces'; import { getWorkspaceSymbolsFromBrsFile } from './symbolUtils'; +import util from '../../util'; export class WorkspaceSymbolProcessor { public constructor( @@ -12,17 +14,30 @@ export class WorkspaceSymbolProcessor { public process() { const results = Object.values(this.event.program.files).map(file => { - if (isBrsFile(file)) { + if (isBrsFile(file) && !file.isSynthetic) { return this.getBrsFileWorkspaceSymbols(file); + } else if (isXmlFile(file)) { + return this.getXmlFileWorkspaceSymbols(file); } return []; }); return results.flat(); } - private getBrsFileWorkspaceSymbols(file: BrsFile) { - const symbols = getWorkspaceSymbolsFromBrsFile(file); + private getBrsFileWorkspaceSymbols(file: BrsFile, uriOverride?: string) { + const symbols = getWorkspaceSymbolsFromBrsFile(file, uriOverride); this.event.workspaceSymbols.push(...symbols); return this.event.workspaceSymbols; } + + private getXmlFileWorkspaceSymbols(file: XmlFile) { + const xmlUri = util.pathToUri(file.srcPath); + for (const pkgPath of file.inlineScriptPkgPaths) { + const brsFile = this.event.program.getFile(pkgPath); + if (brsFile) { + this.getBrsFileWorkspaceSymbols(brsFile, xmlUri); + } + } + return this.event.workspaceSymbols; + } } diff --git a/src/bscPlugin/symbols/symbolUtils.ts b/src/bscPlugin/symbols/symbolUtils.ts index 416ff6c81..3dcdfae3e 100644 --- a/src/bscPlugin/symbols/symbolUtils.ts +++ b/src/bscPlugin/symbols/symbolUtils.ts @@ -28,9 +28,9 @@ export function getDocumentSymbolsFromBrsFile(file: BrsFile) { } } -export function getWorkspaceSymbolsFromBrsFile(file: BrsFile) { +export function getWorkspaceSymbolsFromBrsFile(file: BrsFile, uriOverride?: string) { const result: WorkspaceSymbol[] = []; - const uri = util.pathToUri(file.srcPath); + const uri = uriOverride ?? util.pathToUri(file.srcPath); let symbolsToProcess = getSymbolsFromAstNode(file.ast); while (symbolsToProcess.length > 0) { //get the symbol diff --git a/src/files/BrsFile.ts b/src/files/BrsFile.ts index d8b86cf45..54816f880 100644 --- a/src/files/BrsFile.ts +++ b/src/files/BrsFile.ts @@ -16,6 +16,8 @@ import { Parser, ParseMode } from '../parser/Parser'; import type { FunctionExpression, VariableExpression } from '../parser/Expression'; import type { ClassStatement, NamespaceStatement, AssignmentStatement, MethodStatement, FieldStatement } from '../parser/Statement'; import type { Program } from '../Program'; +import type { XmlFile } from './XmlFile'; +import type { SGScript } from '../parser/SGTypes'; import { DynamicType } from '../types/DynamicType'; import { FunctionType } from '../types/FunctionType'; import { VoidType } from '../types/VoidType'; @@ -79,6 +81,28 @@ export class BrsFile { this.srcPath = value; } + /** + * When true, this file was generated from inline CDATA content in an XML file and is not backed by a file on disk. + * The file watcher should ignore change events for paths that match a synthetic file. + */ + public isSynthetic = false; + + /** + * The XML file this synthetic BrsFile was extracted from. Only set when `isSynthetic` is true. + */ + public parentXmlFile: XmlFile | undefined; + + /** + * The SGScript AST node whose CDATA block this file was extracted from. Only set when `isSynthetic` is true. + * Provides access to the CDATA range (`cdataScript.cdata.range`), raw text, and script type. + */ + public cdataScript: SGScript | undefined; + + /** + * When true, this file will not be written as a standalone output file during transpile. + */ + public excludeFromOutput = false; + /** * Will this file result in only comment or whitespace output? If so, it can be excluded from the output if that bsconfig setting is enabled. */ @@ -309,9 +333,13 @@ export class BrsFile { /** * Calculate the AST for this file - * @param fileContents the raw source code to parse + * @param fileContents the raw source to parse and store + * @param options optional scan options forwarded to the lexer + * @param options.startLine the zero-indexed line to start position tracking from (used for + * inline CDATA fragments so token ranges are in parent XML coordinate space) + * @param options.startCharacter the zero-indexed character offset on the first line */ - public parse(fileContents: string) { + public parse(fileContents: string, options?: { startLine?: number; startCharacter?: number }) { try { this.fileContents = fileContents; this.diagnostics = []; @@ -326,7 +354,9 @@ export class BrsFile { //tokenize the input file let lexer = this.program.logger.time('debug', ['lexer.lex', chalk.green(this.srcPath)], () => { return Lexer.scan(fileContents, { - includeWhitespace: false + includeWhitespace: false, + startLine: options?.startLine, + startCharacter: options?.startCharacter }); }); diff --git a/src/files/XmlFile.spec.ts b/src/files/XmlFile.spec.ts index e8726a7e8..de3b91396 100644 --- a/src/files/XmlFile.spec.ts +++ b/src/files/XmlFile.spec.ts @@ -1,19 +1,20 @@ import { assert, expect } from '../chai-config.spec'; +import { SourceMapConsumer } from 'source-map'; import * as path from 'path'; import * as sinonImport from 'sinon'; import type { CompletionItem } from 'vscode-languageserver'; -import { CompletionItemKind, Position, Range } from 'vscode-languageserver'; +import { CompletionItemKind, Position, Range, SymbolKind } from 'vscode-languageserver'; import * as fsExtra from 'fs-extra'; import { DiagnosticMessages } from '../DiagnosticMessages'; import type { BsDiagnostic, FileReference } from '../interfaces'; import { Program } from '../Program'; import { BrsFile } from './BrsFile'; import { XmlFile } from './XmlFile'; -import { standardizePath as s } from '../util'; +import util, { standardizePath as s } from '../util'; import { expectDiagnostics, expectZeroDiagnostics, getTestTranspile, trim, trimMap } from '../testHelpers.spec'; import { ProgramBuilder } from '../ProgramBuilder'; import { LogLevel } from '../logging'; -import { isXmlFile } from '../astUtils/reflection'; +import { isBrsFile, isXmlFile } from '../astUtils/reflection'; import { tempDir, rootDir, stagingDir } from '../testHelpers.spec'; describe('XmlFile', () => { @@ -1324,4 +1325,854 @@ describe('XmlFile', () => { expect(program.getComponent('comp1')!.file.pkgPath).to.equal(comp2.pkgPath); }); }); + + describe('inline CDATA scripts', () => { + + it('registers a synthetic BrsFile for a brightscript CDATA block', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(xmlFile.inlineScriptPkgPaths).to.eql(['components/MyComp.cdata-0.script.brs']); + const syntheticFile = program.getFile('components/MyComp.cdata-0.script.brs'); + expect(syntheticFile).to.exist; + expect(syntheticFile.isSynthetic).to.be.true; + }); + + it('registers a synthetic .bs file when type is text/brighterscript', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(xmlFile.inlineScriptPkgPaths).to.eql(['components/MyComp.cdata-0.script.bs']); + expect(program.getFile('components/MyComp.cdata-0.script.bs')).to.exist; + }); + + it('registers a synthetic .bs file when type is text/bs', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(xmlFile.inlineScriptPkgPaths).to.eql(['components/MyComp.cdata-0.script.bs']); + expect(program.getFile('components/MyComp.cdata-0.script.bs')).to.exist; + }); + + it('does NOT treat text/brightscript as brighterscript', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(xmlFile.inlineScriptPkgPaths[0]).to.equal('components/MyComp.cdata-0.script.brs'); + }); + + it('indexes multiple CDATA blocks independently', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + + `); + expect(xmlFile.inlineScriptPkgPaths).to.eql([ + 'components/MyComp.cdata-0.script.brs', + 'components/MyComp.cdata-1.script.bs' + ]); + expect(program.getFile('components/MyComp.cdata-0.script.brs')).to.exist; + expect(program.getFile('components/MyComp.cdata-1.script.bs')).to.exist; + }); + + it('synthetic BrsFile token ranges start at the CDATA position in the XML file', () => { + // The ('components/MyComp.xml', trim` + + + + + `); + const brsFile = program.getFile(xmlFile.inlineScriptPkgPaths[0]); + // line 2 is where { + // fileContents stores only the raw CDATA text. Consumers that need line-indexed + // access at XML-space coordinates (e.g. SignatureHelpUtil) use parentXmlFile.fileContents. + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + const brsFile = program.getFile(xmlFile.inlineScriptPkgPaths[0]); + // fileContents is the raw CDATA source, not padded with leading newlines + expect(brsFile.fileContents).to.include('sub greet'); + expect(brsFile.fileContents.split(/\r?\n/g)[0]).to.equal(''); // first "line" is the \n right after { + // Tests that SignatureHelpUtil correctly extracts the function label from the parent + // XML file's contents when the callee is defined in a CDATA block (synthetic BrsFile). + program.setFile('components/MyComp.brs', trim` + sub main() + greet("world") + end sub + `); + program.setFile('components/MyComp.xml', trim` + + + + + `); + program.validate(); + expectZeroDiagnostics(program); + // line 1 of MyComp.brs is " greet("world")" — col 10 is inside the arg list + const help = program.getSignatureHelp('components/MyComp.brs', util.createPosition(1, 10)); + expect(help).to.have.length.greaterThan(0); + expect(help[0].signature.label).to.equal('sub greet(name as string)'); + }); + + it('transpile preserves CDATA blocks inline in the xml output', () => { + testTranspile(trim` + + + + + `, trim` + + + + + + `, trim` + + + + + + `, trim` + + + + + + `, trim` + + + + + + `, trim` + + + + + + `, trim` + + + + + + `, trim` + + + + + + `); + program.options.sourceMap = true; + program.validate(); + const result = xmlFile.transpile(); + expect(result.map).to.exist; + + await SourceMapConsumer.with(result.map.toJSON(), null, (consumer) => { + // raw CDATA lines map at col 0 back to the same line in the xml source + const subPos = consumer.originalPositionFor({ line: 4, column: 0 }); + expect(subPos.source).to.include('MyComp.xml'); + expect(subPos.line).to.equal(4); + expect(subPos.column).to.equal(0); + + const endSubPos = consumer.originalPositionFor({ line: 5, column: 0 }); + expect(endSubPos.source).to.include('MyComp.xml'); + expect(endSubPos.line).to.equal(5); + expect(endSubPos.column).to.equal(0); + }); + }); + + it('source map points back to xml positions for transpiled brighterscript CDATA', async () => { + // Generated output (1-based lines): + // 3: " + + `); + program.options.sourceMap = true; + program.validate(); + const result = xmlFile.transpile(); + expect(result.map).to.exist; + // only the xml file should appear in sources — not the synthetic brs file + expect(result.map.toJSON().sources).to.eql([s`${rootDir}/components/MyComp.xml`]); + + await SourceMapConsumer.with(result.map.toJSON(), null, (consumer) => { + // 'sub' is on gen line 3 right after ' { + // Generated output (1-based lines): + // 5: "sub init()" ← col 0 → source (7, 8) + // 6: " d = \"up\"" ← col 4 → source (8, 12) + // 7: "end sub]]>..." ← col 0 → source (9, 8) + const xmlFile = program.setFile({ src: s`${rootDir}/components/MyComp.xml`, dest: 'components/MyComp.xml' }, trim` + + + + + `); + program.options.sourceMap = true; + program.validate(); + const result = xmlFile.transpile(); + expect(result.map).to.exist; + expect(result.map.toJSON().sources).to.eql([s`${rootDir}/components/MyComp.xml`]); + + await SourceMapConsumer.with(result.map.toJSON(), null, (consumer) => { + // 'sub' on gen line 5 col 0 → source (7, 8) + const subPos = consumer.originalPositionFor({ line: 5, column: 0 }); + expect(subPos.source).to.include('MyComp.xml'); + expect(subPos.line).to.equal(7); + expect(subPos.column).to.equal(8); + + // 'd' on gen line 6 col 4 → source (8, 12) + const dPos = consumer.originalPositionFor({ line: 6, column: 4 }); + expect(dPos.source).to.include('MyComp.xml'); + expect(dPos.line).to.equal(8); + expect(dPos.column).to.equal(12); + }); + }); + + it('cleans up synthetic files when the xml file is removed', () => { + program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(program.getFile('components/MyComp.cdata-0.script.brs')).to.exist; + program.removeFile(s`${rootDir}/components/MyComp.xml`); + expect(program.getFile('components/MyComp.cdata-0.script.brs')).to.not.exist; + }); + + it('replaces old synthetic files when the xml file is re-parsed', () => { + program.setFile('components/MyComp.xml', trim` + + + + + `); + const firstFile = program.getFile('components/MyComp.cdata-0.script.brs'); + expect(firstFile).to.exist; + + //re-set with different cdata content + program.setFile('components/MyComp.xml', trim` + + + + + `); + const secondFile = program.getFile('components/MyComp.cdata-0.script.brs'); + expect(secondFile).to.exist; + //a new BrsFile object should have been created + expect(secondFile).to.not.equal(firstFile); + }); + + it('includes the synthetic file in scope dependencies', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + const deps = xmlFile.getOwnDependencies(); + expect(deps.some(d => d.includes('cdata-0.script.brs'))).to.be.true; + }); + + it('validates code inside CDATA blocks and remaps diagnostics to the xml file', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + program.validate(); + const allDiagnostics = program.getDiagnostics(); + //diagnostics from the CDATA block should be remapped to the parent xml file, not the synthetic file + expect(allDiagnostics.some(d => d.file.pkgPath === xmlFile.pkgPath)).to.be.true; + expect(allDiagnostics.every(d => d.file.pkgPath !== 'components/MyComp.cdata-0.script.brs')).to.be.true; + //the reported position should be within the xml file's line range (not line 1 of a standalone brs file) + const cdataDiagnostics = allDiagnostics.filter(d => d.file.pkgPath === xmlFile.pkgPath); + expect(cdataDiagnostics.every(d => d.range.start.line > 2)).to.be.true; + }); + + it('sets needsTranspiled on the xml file when CDATA is present', () => { + const xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + expect(xmlFile.needsTranspiled).to.be.true; + }); + + describe('LSP event remapping', () => { + // After trim, the file looks like: + // line 0: + // line 1: + // line 2: + // line 10: + const cdataStartLine = 2; + + let xmlFile: XmlFile; + beforeEach(() => { + xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + program.validate(); + }); + + it('getCompletions returns BrightScript completions inside CDATA', () => { + // position cursor at `name` usage on line 4 col 20 (inside "name") + const completions = program.getCompletions(xmlFile.srcPath, Position.create(4, 20)); + expect(completions.map(c => c.label)).to.include('name'); + }); + + it('getCompletions returns empty outside CDATA', () => { + // position inside the tag (line 1), not in CDATA + const completions = program.getCompletions(xmlFile.srcPath, Position.create(1, 5)); + expect(completions).to.be.empty; + }); + + it('getHover returns hover info for a function inside CDATA', () => { + // hover over `greet` at line 3 col 12 + const hovers = program.getHover(xmlFile.srcPath, Position.create(3, 12)); + expect(hovers).to.have.length.greaterThan(0); + }); + + it('getHover remaps the hover range to XML coordinates', () => { + // hover over `greet` at line 3 col 12 + const hovers = program.getHover(xmlFile.srcPath, Position.create(3, 12)); + for (const hover of hovers) { + if (hover.range) { + // range must be within the CDATA block in XML coordinates, not synthetic line 0 + expect(hover.range.start.line).to.be.at.least(cdataStartLine); + } + } + }); + + it('getDefinition returns a location inside the XML file for a symbol in CDATA', () => { + // go-to-definition on `greet` reference (line 3, col 12 is the definition site itself) + // use getCount call from synthetic line 0 edge case — instead hover greet on its def line + const locations = program.getDefinition(xmlFile.srcPath, Position.create(3, 12)); + expect(locations).to.have.length.greaterThan(0); + for (const loc of locations) { + // all returned locations must use the xml file URI + expect(loc.uri).to.include('MyComp.xml'); + // and be within the CDATA block line range + expect(loc.range.start.line).to.be.at.least(cdataStartLine); + } + }); + + it('getReferences returns locations in XML coordinates', () => { + // find references of the `name` parameter — position inside the word (not at the start) + // line 3: ` sub greet(name as string)` — `name` starts at col 18, use col 20 (inside) + const refs = program.getReferences(xmlFile.srcPath, Position.create(3, 20)); + expect(refs).to.have.length.greaterThan(0); + for (const ref of refs) { + expect(ref.uri).to.include('MyComp.xml'); + expect(ref.range.start.line).to.be.at.least(cdataStartLine); + } + }); + + it('getSemanticTokens returns tokens with XML-file coordinates', () => { + // Use a .bs CDATA block with namespace+class to generate semantic tokens + const bsXmlFile = program.setFile('components/BsComp.xml', trim` + + + + + `); + program.validate(); + const tokens = program.getSemanticTokens(bsXmlFile.srcPath); + expect(tokens).to.exist; + expect(tokens).to.have.length.greaterThan(0); + for (const token of tokens) { + // all tokens must be inside the CDATA block (line 2+ in XML) + expect(token.range.start.line).to.be.at.least(cdataStartLine + 1); + } + }); + + it('getDocumentSymbols returns function symbols with XML-file coordinates', () => { + const symbols = program.getDocumentSymbols(xmlFile.srcPath); + expect(symbols).to.exist; + const names = symbols.map(s => s.name); + expect(names).to.include('greet'); + expect(names).to.include('getCount'); + for (const sym of symbols) { + if (sym.name === 'greet' || sym.name === 'getCount') { + expect(sym.kind).to.equal(SymbolKind.Function); + // symbol range must start inside the CDATA block + expect(sym.range.start.line).to.be.at.least(cdataStartLine + 1); + } + } + }); + + it('getSignatureHelp returns signature for a function call inside CDATA', () => { + // Add a call site so we can test signature help + xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + program.validate(); + // line 6 is ` greet(` — `(` is at col 17, col 18 is inside the arg list + const sigHelp = program.getSignatureHelp(xmlFile.srcPath, Position.create(6, 18)); + expect(sigHelp).to.have.length.greaterThan(0); + expect(sigHelp[0].signature.label).to.include('greet'); + }); + + it('getCodeActions workspace edits reference the XML file URI', () => { + // use a range inside the CDATA block (line 3 = `sub greet(name as string)`) + const actions = program.getCodeActions(xmlFile.srcPath, Range.create(3, 8, 3, 30)); + // if any code action has workspace edits, they must target the xml file + for (const action of actions) { + if (action.edit?.changes) { + for (const uri of Object.keys(action.edit.changes)) { + expect(uri).to.include('MyComp.xml'); + } + } + if (action.edit?.documentChanges) { + for (const change of action.edit.documentChanges) { + if ('textDocument' in change) { + expect(change.textDocument.uri).to.include('MyComp.xml'); + } + } + } + } + }); + + it('hover works when CDATA content starts on the same line as the opening marker', () => { + // The synthetic BrsFile is padded so its positions match XML coordinates directly. + // When content starts inline with ('components/Inline.xml', trim` + + + + + `); + program.validate(); + // ` { + // Verifies that emitWithSyntheticFileContext sets _cdataDiagnosticsContext for + // every plugin event that fires with a synthetic BrsFile as event.file, so that + // plugins calling program.getDiagnostics() from inside the handler get results + // where x.file === event.file holds (diagnostics not yet remapped to the XmlFile). + + let xmlFile: XmlFile; + + beforeEach(() => { + xmlFile = program.setFile('components/MyComp.xml', trim` + + + + + `); + program.validate(); + }); + + // Note: afterFileParse cannot be covered here because isSynthetic is set on the + // BrsFile *after* setFile() returns, which is after afterFileParse fires. So the + // emitWithSyntheticFileContext wrapper cannot identify the file as synthetic at + // that point. This is a known limitation. + + it('onFileValidate: _cdataDiagnosticsContext is set to the synthetic BrsFile during the event', () => { + // onFileValidate fires during the validation pass, before scope diagnostics are + // produced. We verify the context flag is set so getDiagnostics() would scope + // correctly if called by a plugin. + let contextFileInsideHandler: BrsFile | undefined; + program.plugins.add({ + name: 'test', + onFileValidate: function(event) { + if (isBrsFile(event.file) && event.file.isSynthetic) { + contextFileInsideHandler = (program as any)._cdataDiagnosticsContext; + } + } + }); + program.setFile('components/MyComp.xml', xmlFile.fileContents); + program.validate(); + expect(contextFileInsideHandler).to.exist; + expect((contextFileInsideHandler as BrsFile).isSynthetic).to.be.true; + }); + + it('provideDocumentSymbols: program.getDiagnostics() inside handler associates diagnostics with the synthetic BrsFile', () => { + let fileIdentityWorked = false; + program.plugins.add({ + name: 'test', + provideDocumentSymbols: function(event) { + if (isBrsFile(event.file) && (event.file as BrsFile).isSynthetic) { + const diags = program.getDiagnostics() + .filter(x => x.file === event.file); + if (diags.length > 0) { + fileIdentityWorked = true; + } + } + } + }); + program.getDocumentSymbols(xmlFile.srcPath); + expect(fileIdentityWorked).to.be.true; + }); + + it('onGetSemanticTokens: program.getDiagnostics() inside handler associates diagnostics with the synthetic BrsFile', () => { + let fileIdentityWorked = false; + program.plugins.add({ + name: 'test', + onGetSemanticTokens: function(event) { + if (isBrsFile(event.file) && (event.file as BrsFile).isSynthetic) { + const diags = program.getDiagnostics() + .filter(x => x.file === event.file); + if (diags.length > 0) { + fileIdentityWorked = true; + } + } + } + }); + program.getSemanticTokens(xmlFile.srcPath); + expect(fileIdentityWorked).to.be.true; + }); + + it('onGetCodeActions: program.getDiagnostics() inside handler supports "fix all" pattern across multiple CDATA diagnostics', () => { + // Two `then` violations in one CDATA block — simulate the bslint "fix all" pattern + const twoThenFile = program.setFile('components/TwoThen.xml', trim` + + + + + `); + program.validate(); + const twoThenSynthetic = program.getFile('components/TwoThen.cdata-0.script.brs'); + + // Simulate plugin calling getDiagnostics() filtered by event.file identity + const fixAllDiagsFoundBySyntheticIdentity: BsDiagnostic[][] = []; + program.plugins.add({ + name: 'test', + onGetCodeActions: function(event) { + if (event.file === twoThenSynthetic) { + // This is exactly the pattern a "fix all" plugin would use + const allInFile = program.getDiagnostics() + .filter(x => x.file === event.file); + fixAllDiagsFoundBySyntheticIdentity.push(allInFile); + } + } + }); + + // trigger inside first function range + program.getCodeActions(twoThenFile.srcPath, Range.create(3, 12, 3, 12)); + + // the plugin must have seen diagnostics associated with the synthetic BrsFile + expect(fixAllDiagsFoundBySyntheticIdentity.length).to.be.greaterThan(0); + // and all returned diagnostics must reference the synthetic file, not the xml file + for (const group of fixAllDiagsFoundBySyntheticIdentity) { + for (const d of group) { + expect(d.file).to.equal(twoThenSynthetic, 'expected diagnostic.file to be the synthetic BrsFile, not the XmlFile'); + } + } + }); + }); + }); }); + diff --git a/src/files/XmlFile.ts b/src/files/XmlFile.ts index e3172998a..239ba7eb1 100644 --- a/src/files/XmlFile.ts +++ b/src/files/XmlFile.ts @@ -6,6 +6,7 @@ import { DiagnosticCodeMap, diagnosticCodes } from '../DiagnosticMessages'; import type { FunctionScope } from '../FunctionScope'; import type { Callable, BsDiagnostic, File, FileReference, FunctionCall, CommentFlag } from '../interfaces'; import type { Program } from '../Program'; +import type { BrsFile } from './BrsFile'; import util from '../util'; import SGParser, { rangeFromTokenValue } from '../parser/SGParser'; import chalk from 'chalk'; @@ -75,6 +76,17 @@ export class XmlFile { */ readonly canBePruned = false; + /** + * XML files are never synthetically generated; this is always false. + * Exists to satisfy the BscFile union type alongside BrsFile.isSynthetic. + */ + readonly isSynthetic = false; + + /** + * When true, this file will not be written as a standalone output file during transpile. + */ + public excludeFromOutput = false; + /** * The list of script imports delcared in the XML of this file. * This excludes parent imports and auto codebehind imports @@ -168,6 +180,24 @@ export class XmlFile { //TODO implement the xml CDATA parsing, which would populate this list public functionCalls = [] as FunctionCall[]; + /** + * The pkg paths of synthetic BrsFiles generated from inline CDATA script blocks in this file. + * Populated during `parse()`. Used by Program.setFile() to register the files, by + * attachDependencyGraph() to add them as dependencies, by transpile() to replace CDATA + * with uri script tags, and by dispose() to clean them up. + */ + public inlineScriptPkgPaths: string[] = []; + + /** + * Derive the pkg path for a synthetic file extracted from a CDATA block. + * Format: `components/MyComp.cdata-0.script.brs` (or `.bs` for brighterscript type). + */ + private getInlineScriptPkgPath(cdataIndex: number, scriptType: string | undefined): string { + const isBrighterScript = /brighterscript|text\/bs\b/i.test(scriptType ?? ''); + const ext = isBrighterScript ? '.bs' : '.brs'; + return this.pkgPath.replace(/\\/g, '/').replace(/\.xml$/i, `.cdata-${cdataIndex}.script${ext}`); + } + public functionScopes = [] as FunctionScope[]; /** @@ -209,6 +239,7 @@ export class XmlFile { */ public parse(fileContents: string) { this.fileContents = fileContents; + this.inlineScriptPkgPaths = []; this.parser.parse(this.pkgPath, fileContents); this.diagnostics = this.parser.diagnostics.map(diagnostic => ({ @@ -222,6 +253,16 @@ export class XmlFile { this.needsTranspiled = this.needsTranspiled || this.ast.component?.scripts?.some( script => script.type?.indexOf('brighterscript') > 0 || script.uri?.endsWith('.bs') ); + + //collect inline CDATA scripts and assign them synthetic pkg paths + let cdataIndex = 0; + for (const script of this.ast.component?.scripts ?? []) { + if (script.cdata) { + this.inlineScriptPkgPaths.push(this.getInlineScriptPkgPath(cdataIndex++, script.type)); + //any CDATA script requires the XML to be rewritten (CDATA → uri tag) + this.needsTranspiled = true; + } + } } /** @@ -270,7 +311,8 @@ export class XmlFile { }); let dependencies = [ - ...this.scriptTagImports.map(x => x.pkgPath.toLowerCase()) + ...this.scriptTagImports.map(x => x.pkgPath.toLowerCase()), + ...this.inlineScriptPkgPaths.map(p => util.standardizePath(p).toLowerCase()) ]; //if autoImportComponentScript is enabled, add the .bs and .brs files with the same name if (this.program.options.autoImportComponentScript) { @@ -434,11 +476,17 @@ export class XmlFile { return map; }, {}); - //if the XML already has this import, skip this one - let alreadyThereScriptImportMap = this.scriptTagImports.reduce((map, fileReference) => { + //if the XML already has this import (either a uri script tag or an inline CDATA script), skip this one + let alreadyThereScriptImportMap = this.scriptTagImports.reduce>((map, fileReference) => { map[fileReference.pkgPath.toLowerCase()] = true; return map; }, {}); + for (const pkgPath of this.inlineScriptPkgPaths) { + const normalizedPkgPath = util.standardizePath(pkgPath).toLowerCase(); + alreadyThereScriptImportMap[normalizedPkgPath] = true; + //also mark the .brs variant so a .bs inline script doesn't get added as an extra import + alreadyThereScriptImportMap[normalizedPkgPath.replace(/\.bs$/, '.brs')] = true; + } let resultMap = {}; let result = [] as string[]; @@ -489,6 +537,11 @@ export class XmlFile { const state = new TranspileState(this.srcPath, this.program.options); const originalScripts = this.ast.component?.scripts ?? []; + + const anySyntheticNeedsTranspiled = this.inlineScriptPkgPaths.some( + pkgPath => this.program.getFile(pkgPath)?.needsTranspiled + ); + const extraImportScripts = this.getMissingImportsForTranspile().map(uri => { const script = new SGScript(); script.uri = util.getRokuPkgPath(uri.replace(/\.bs$/, '.brs')); @@ -501,11 +554,10 @@ export class XmlFile { ]); let transpileResult: SourceNode | undefined; - if (this.needsTranspiled || extraImportScripts.length > 0 || scriptsHaveChanged) { + if (this.needsTranspiled || anySyntheticNeedsTranspiled || extraImportScripts.length > 0 || scriptsHaveChanged) { //temporarily add the missing imports as script tags this.ast.component.scripts = publishableScripts; - transpileResult = util.sourceNodeFromTranspileResult(null, null, state.srcPath, this.parser.ast.transpile(state)); //restore the original scripts array @@ -537,5 +589,9 @@ export class XmlFile { public dispose() { //unsubscribe from any DependencyGraph subscriptions this.unsubscribeFromDependencyGraph?.(); + //clean up any synthetic BrsFiles that were generated from CDATA blocks in this file + for (const pkgPath of this.inlineScriptPkgPaths) { + this.program.removeFile(pkgPath); + } } } diff --git a/src/interfaces.ts b/src/interfaces.ts index 79137bb1d..758266c79 100644 --- a/src/interfaces.ts +++ b/src/interfaces.ts @@ -319,17 +319,27 @@ export interface Plugin { afterScopeValidate?: ValidateHandler; //file events beforeFileParse?: (source: SourceObj) => void; + /** + * Called after each file has been parsed. When a SceneGraph XML file contains inline + * `` script blocks, this event is also fired once per CDATA block with a + * synthetic `BrsFile` as the argument (`file.isSynthetic === true`). The synthetic file's + * positions are already in the parent XML file's coordinate space. + */ afterFileParse?: (file: BscFile) => void; /** - * Called before each file is validated + * Called before each file is validated. For SceneGraph XML files with inline CDATA script + * blocks, this is also called for each synthetic `BrsFile` (`file.isSynthetic === true`). */ beforeFileValidate?: PluginHandler; /** * Called during the file validation process. If your plugin contributes file validations, this is a good place to contribute them. + * For SceneGraph XML files with inline CDATA script blocks, this is also called for each + * synthetic `BrsFile` (`event.file.isSynthetic === true`). */ onFileValidate?: PluginHandler; /** - * Called after each file is validated + * Called after each file is validated. For SceneGraph XML files with inline CDATA script + * blocks, this is also called for each synthetic `BrsFile` (`file.isSynthetic === true`). */ afterFileValidate?: (file: BscFile) => void; beforeFileTranspile?: PluginHandler; @@ -341,6 +351,11 @@ export type PluginHandler = (event: T) => R; export interface OnGetCodeActionsEvent { program: Program; + /** + * The file the code action request was invoked in. When the range falls inside a + * `` block of an XML file, this will be the synthetic `BrsFile` for that + * block (`file.isSynthetic === true`) rather than the `XmlFile` itself. + */ file: BscFile; range: Range; scopes: Scope[]; @@ -350,6 +365,11 @@ export interface OnGetCodeActionsEvent { export interface ProvideCompletionsEvent { program: Program; + /** + * The file the completion request was invoked in. When the cursor position falls inside a + * `` block of an XML file, this will be the synthetic `BrsFile` for that + * block (`file.isSynthetic === true`) rather than the `XmlFile` itself. + */ file: TFile; scopes: Scope[]; position: Position; @@ -360,6 +380,11 @@ export type AfterProvideCompletionsEvent = Prov export interface ProvideHoverEvent { program: Program; + /** + * The file the hover request was invoked in. When the cursor position falls inside a + * `` block of an XML file, this will be the synthetic `BrsFile` for that + * block (`file.isSynthetic === true`) rather than the `XmlFile` itself. + */ file: BscFile; position: Position; scopes: Scope[]; @@ -386,7 +411,9 @@ export type AfterProvideHoverEvent = ProvideHoverEvent; export interface ProvideDefinitionEvent { program: Program; /** - * The file that the getDefinition request was invoked in + * The file that the getDefinition request was invoked in. When the cursor position falls inside + * a `` block of an XML file, this will be the synthetic `BrsFile` for that + * block (`file.isSynthetic === true`) rather than the `XmlFile` itself. */ file: TFile; /** @@ -404,7 +431,9 @@ export type AfterProvideDefinitionEvent = ProvideDefinitionEven export interface ProvideReferencesEvent { program: Program; /** - * The file that the getDefinition request was invoked in + * The file that the getReferences request was invoked in. When the cursor position falls inside + * a `` block of an XML file, this will be the synthetic `BrsFile` for that + * block (`file.isSynthetic === true`) rather than the `XmlFile` itself. */ file: TFile; /** @@ -423,7 +452,10 @@ export type AfterProvideReferencesEvent = ProvideReferencesEven export interface ProvideDocumentSymbolsEvent { program: Program; /** - * The file that the `documentSymbol` request was invoked in + * The file that the `documentSymbol` request was invoked in. For SceneGraph XML files with + * inline CDATA script blocks, this event is also emitted once per block with the synthetic + * `BrsFile` (`file.isSynthetic === true`). Symbol ranges are already in the parent XML + * coordinate space. */ file: TFile; /** @@ -452,7 +484,9 @@ export interface OnGetSemanticTokensEvent { */ program: Program; /** - * The file to get semantic tokens for + * The file to get semantic tokens for. For SceneGraph XML files with inline CDATA script + * blocks, this event is also emitted once per block with the synthetic `BrsFile` + * (`file.isSynthetic === true`). Token ranges are already in the parent XML coordinate space. */ file: T; /** diff --git a/src/lexer/Lexer.spec.ts b/src/lexer/Lexer.spec.ts index 6dfaf2c3f..e4bfdaccf 100644 --- a/src/lexer/Lexer.spec.ts +++ b/src/lexer/Lexer.spec.ts @@ -1225,6 +1225,40 @@ describe('lexer', () => { Range.create(2, 7, 2, 8) // EOF ]); }); + + it('offsets all token ranges by startLine and startCharacter', () => { + // Simulates scanning an inline CDATA fragment that begins on line 5, col 10 of a parent file. + // The first character of the source is a newline (right after ({ kind: t.kind, range: t.range }))).to.eql([ + { kind: TokenKind.Newline, range: Range.create(5, 10, 5, 11) }, + { kind: TokenKind.Sub, range: Range.create(6, 0, 6, 3) }, + { kind: TokenKind.Identifier, range: Range.create(6, 4, 6, 7) }, // foo + { kind: TokenKind.LeftParen, range: Range.create(6, 7, 6, 8) }, + { kind: TokenKind.RightParen, range: Range.create(6, 8, 6, 9) }, + { kind: TokenKind.Newline, range: Range.create(6, 9, 6, 10) }, + { kind: TokenKind.EndSub, range: Range.create(7, 0, 7, 7) }, + { kind: TokenKind.Eof, range: Range.create(7, 7, 7, 8) } + ]); + }); + + it('produces no tokens before startLine when scanning a fragment with a leading newline', () => { + // The leading newline is at the startLine position — no tokens at lines 0..startLine-1. + const { tokens } = Lexer.scan('\nsub foo()\nend sub', { + includeWhitespace: false, + startLine: 5, + startCharacter: 10 + }); + for (const token of tokens) { + expect(token.range.start.line).to.be.at.least(5); + } + }); }); describe('two word keywords', () => { diff --git a/src/lexer/Lexer.ts b/src/lexer/Lexer.ts index 6f45df185..b6c9b4960 100644 --- a/src/lexer/Lexer.ts +++ b/src/lexer/Lexer.ts @@ -93,10 +93,10 @@ export class Lexer { this.options = this.sanitizeOptions(options); this.start = 0; this.current = 0; - this.lineBegin = 0; - this.lineEnd = 0; - this.columnBegin = 0; - this.columnEnd = 0; + this.lineBegin = this.options.startLine ?? 0; + this.lineEnd = this.options.startLine ?? 0; + this.columnBegin = this.options.startCharacter ?? 0; + this.columnEnd = this.options.startCharacter ?? 0; this.tokens = []; this.diagnostics = []; while (!this.isAtEnd()) { @@ -1102,4 +1102,17 @@ export interface ScanOptions { * @default true */ trackLocations?: boolean; + /** + * The zero-indexed line number to start position tracking from. + * Useful when scanning a fragment of a larger file (e.g. an inline CDATA block) + * so that all token ranges are in the coordinate space of the parent file. + * @default 0 + */ + startLine?: number; + /** + * The zero-indexed character offset to start position tracking from on the first line. + * Only applies to the first line; subsequent lines always start at column 0. + * @default 0 + */ + startCharacter?: number; } diff --git a/src/parser/SGParser.spec.ts b/src/parser/SGParser.spec.ts index 22f7f451e..743db90a9 100644 --- a/src/parser/SGParser.spec.ts +++ b/src/parser/SGParser.spec.ts @@ -158,4 +158,70 @@ describe('SGParser', () => { range: Range.create(0, 0, 1, 12) }); }); + + describe('SGScript.cdataText', () => { + function parseScript(xml: string) { + const parser = new SGParser(); + parser.parse('pkg:/components/Comp.xml', xml); + return parser.ast.component?.scripts?.[0]; + } + + it('strips from multiline content', () => { + const script = parseScript(trim` + + + + + `); + // cdataText should not contain the delimiters; whitespace inside is preserved as-is + expect(script?.cdataText).to.not.include(''); + expect(script?.cdataText?.trim()).to.equal('function init()\n end function'); + }); + + it('strips from inline (single-line) content', () => { + const script = parseScript(trim` + + + + + `); + expect(script?.cdataText).to.equal('function init() : end function'); + }); + + it('returns empty string for empty CDATA block', () => { + const script = parseScript(trim` + + + + + `); + expect(script?.cdataText).to.equal(''); + }); + + it('preserves > characters inside CDATA content', () => { + const script = parseScript(trim` + + + + + `); + expect(script?.cdataText).to.equal('if x > 5 then : end if'); + }); + + it('returns undefined when there is no CDATA block', () => { + const script = parseScript(trim` + + +