diff --git a/AGENTS.md b/AGENTS.md index 35ce28b..a326e55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,6 +23,7 @@ - Compiler arguments are parsed with the repository's POSIX-like tokenizer. Preserve quoting, escaped paths, command-style output, and repeated macro arguments. - Relevant malformed compiler, `cd`, and Make commands emit a recoverable Error diagnostic containing the build-log line, tracked cwd, reason, and byte offset without the full command. Unrelated malformed output remains silent; parser status stays successful unless canceled. - Complex shell groups, functions, and control structures are skipped as a whole. Backslash-newline continuation follows shell joining semantics, does not insert whitespace, and drops malformed or unterminated fragments without changing the successful parser status. +- A top-level redirected simple command that cannot change tracked state is skipped without discarding independent `;` siblings. Redirected state-changing commands and redirections inside conditionals or complex structures remain fail closed for the logical line. ## Logging And Output diff --git a/internal/make_wrap.go b/internal/make_wrap.go index d5902f4..6ded252 100644 --- a/internal/make_wrap.go +++ b/internal/make_wrap.go @@ -463,17 +463,44 @@ func isConflictingLongMakeOption(argument string) bool { name == "no-print-directory" || name == "print-data-base" || name == "help" || name == "version" } -func makeLongOptionTakesArgument(argument string) bool { +type makeOptionArgumentMode uint8 + +const ( + makeOptionArgumentNone makeOptionArgumentMode = iota + makeOptionArgumentRequired + makeOptionArgumentOptionalAttached +) + +func makeLongOptionArgumentMode(argument string) makeOptionArgumentMode { name, ok := canonicalMakeLongOption(argument) if !ok { - return false + return makeOptionArgumentNone } switch name { case "directory", "file", "makefile", "include-dir", "eval", "old-file", "assume-old", - "new-file", "assume-new", "what-if": - return true + "new-file", "assume-new", "what-if", "jobserver-auth", "jobserver-fds", "jobserver-style": + return makeOptionArgumentRequired + case "debug", "jobs", "load-average", "max-load", "output-sync", "shuffle": + return makeOptionArgumentOptionalAttached default: - return false + return makeOptionArgumentNone + } +} + +func makeLongOptionTakesArgument(argument string) bool { + return makeLongOptionArgumentMode(argument) == makeOptionArgumentRequired +} + +func makeShortOptionArgumentMode(option byte) (makeOptionArgumentMode, bool) { + switch { + case strings.ContainsRune("CEfIoW", rune(option)): + return makeOptionArgumentRequired, true + case strings.ContainsRune("jlO", rune(option)): + return makeOptionArgumentOptionalAttached, true + case strings.ContainsRune("bmBdehikLnpqrRsStvw", rune(option)): + return makeOptionArgumentNone, true + default: + return makeOptionArgumentNone, false } } diff --git a/internal/parser.go b/internal/parser.go index 84e2048..d9e1413 100644 --- a/internal/parser.go +++ b/internal/parser.go @@ -22,9 +22,7 @@ var ( RegexFile string = `^.*\s+-c.*\s(?:(?:"|')(.*?\.(?i:c|cpp|cc|cxx|c\+\+|s|m|mm|cu))(?:"|')|([^\s"']+\.(?i:c|cpp|cc|cxx|c\+\+|s|m|mm|cu)))(\s|$)` // We want to skip such lines from configure to avoid spurious MAKE expansion errors. - checkingMake = regexp.MustCompile(`^checking whether .* sets \$\(\w+\)\.\.\. (yes|no)$`) - shellControlCommand = regexp.MustCompile(`(?:^|[;&|]\s*)(?:!\s*)?(?:if|then|elif|else|fi|case|esac|for|while|until|do|done|select|function)(?:\s|$)`) - shellFunction = regexp.MustCompile(`(?:^|[;&|]\s*)[A-Za-z_][A-Za-z0-9_]*\s*\(\s*\)`) + checkingMake = regexp.MustCompile(`^checking whether .* sets \$\(\w+\)\.\.\. (yes|no)$`) ) const maxBuildLogLineSize = 100 * 1024 * 1024 @@ -37,11 +35,6 @@ type parserPatterns struct { defaultFile bool } -type shellCommand struct { - text string - separator string -} - type logicalLine struct { text string line int @@ -232,231 +225,6 @@ func shellLexicalError(line string) *shellTokenizationError { return nil } -func splitShellCommands(line string) ([]shellCommand, bool) { - commands := []shellCommand{} - start := 0 - separator := "" - var quote byte - escaped := false - wordStarted := false - commandSubstitutionDepth := 0 - groupDepth := 0 - flush := func(end int, nextSeparator string) { - if command := strings.TrimSpace(line[start:end]); command != "" { - commands = append(commands, shellCommand{text: command, separator: separator}) - } - separator = nextSeparator - } - - for i := 0; i < len(line); i++ { - character := line[i] - if escaped { - escaped = false - wordStarted = true - continue - } - if quote != 0 { - if character == '\\' && quote != '\'' { - escaped = true - } else if character == quote { - quote = 0 - } - continue - } - if commandSubstitutionDepth > 0 { - if character == '(' { - commandSubstitutionDepth++ - } else if character == ')' { - commandSubstitutionDepth-- - } - continue - } - if character == '$' && i+1 < len(line) && line[i+1] == '(' { - wordStarted = true - commandSubstitutionDepth = 1 - i++ - continue - } - if character == '#' && !wordStarted { - if strings.TrimSpace(line[start:i]) == "" && (separator == "&&" || separator == "||") { - return commands, false - } - flush(i, "") - return commands, true - } - switch character { - case '\\': - escaped = true - wordStarted = true - case '\'', '"', '`': - quote = character - wordStarted = true - case ' ', '\t', '\r', '\n': - wordStarted = false - case '(': - groupDepth++ - wordStarted = false - case ')': - if groupDepth > 0 { - groupDepth-- - } - wordStarted = false - case ';': - if groupDepth > 0 { - wordStarted = false - continue - } - flush(i, ";") - start = i + 1 - wordStarted = false - case '&', '|': - if groupDepth == 0 && i+1 < len(line) && line[i+1] == character { - flush(i, line[i:i+2]) - i++ - start = i + 1 - } - wordStarted = false - case '<', '>': - wordStarted = false - default: - wordStarted = true - } - } - flush(len(line), "") - return commands, true -} - -func hasUnsupportedShellControlStructure(line string) bool { - var quote byte - escaped := false - wordStarted := false - visible := make([]byte, len(line)) - for i := 0; i < len(line); i++ { - character := line[i] - if escaped { - escaped = false - wordStarted = true - visible[i] = ' ' - continue - } - if quote != 0 { - visible[i] = ' ' - if character == '\\' && quote != '\'' { - escaped = true - } else if quote == '"' && character == '$' && i+1 < len(line) && line[i+1] == '(' { - return true - } else if character == quote { - quote = 0 - } - continue - } - if character == '$' && i+1 < len(line) && line[i+1] == '(' { - return true - } - if character == '#' && !wordStarted { - break - } - switch character { - case '\\': - escaped = true - wordStarted = true - visible[i] = ' ' - case '\'', '"', '`': - quote = character - wordStarted = true - visible[i] = ' ' - case ' ', '\t', '\r', '\n', ';', '|', '&', '<', '>', '(', ')': - wordStarted = false - visible[i] = character - case '{', '}': - return true - default: - wordStarted = true - visible[i] = character - } - } - text := string(visible) - return shellControlCommand.MatchString(text) || shellFunction.MatchString(text) -} - -func hasUnsupportedShellSyntax(line string) bool { - var quote byte - escaped := false - inBacktick := false - for i := 0; i < len(line); i++ { - character := line[i] - if escaped { - escaped = false - continue - } - if inBacktick { - if character == '\\' { - escaped = true - } else if character == '`' { - inBacktick = false - } - continue - } - if quote == '\'' { - if character == quote { - quote = 0 - } - continue - } - if quote == '"' { - if character == '\\' { - escaped = true - } else if character == quote { - quote = 0 - } else if character == '`' { - inBacktick = true - } else if character == '$' && i+1 < len(line) && line[i+1] == '(' { - return true - } - continue - } - switch character { - case '\\': - escaped = true - case '\'', '"': - quote = character - case '`': - inBacktick = true - case '<', '>', '(', ')', '{', '}': - return true - case '$': - if i+1 < len(line) && line[i+1] == '(' { - return true - } - case '|', '&': - return true - } - } - return false -} - -func shellCommandExecution(separator string, previous shellCommandStatus) (bool, bool) { - switch separator { - case "", ";": - return true, true - case "&&": - if previous == shellStatusSuccess { - return true, true - } - if previous == shellStatusFailure { - return false, true - } - case "||": - if previous == shellStatusFailure { - return true, true - } - if previous == shellStatusSuccess { - return false, true - } - } - return false, false -} - type logicalLineIssue struct { line int reason string @@ -703,7 +471,87 @@ func isMakeExecutable(argument string) bool { } func isMakeExecutableFromArguments(arguments []string) bool { - return len(arguments) > 0 && isMakeExecutable(arguments[0]) + index := shellExecutableIndex(arguments) + return index < len(arguments) && isMakeExecutable(arguments[index]) +} + +func shellExecutableIndex(arguments []string) int { + index := 0 + for index < len(arguments) && isShellAssignment(arguments[index]) { + index++ + } + return index +} + +type makeArgumentSpec struct { + option bool + stop bool + valid bool + valueMode makeOptionArgumentMode + directory bool + valueInline bool + inlineValue string +} + +func classifyMakeArgument(argument string) makeArgumentSpec { + if argument == "--" { + return makeArgumentSpec{option: true, stop: true, valid: true} + } + if strings.HasPrefix(argument, "--") { + name, ok := canonicalMakeLongOption(argument) + if !ok { + return makeArgumentSpec{} + } + _, value, attached := strings.Cut(argument, "=") + valueMode := makeLongOptionArgumentMode(argument) + if attached && valueMode == makeOptionArgumentNone { + return makeArgumentSpec{} + } + return makeArgumentSpec{ + option: true, + valid: true, + valueMode: valueMode, + directory: name == "directory", + valueInline: attached, + inlineValue: value, + } + } + if len(argument) < 2 || argument[0] != '-' { + return makeArgumentSpec{valid: true} + } + for index := 1; index < len(argument); index++ { + option := argument[index] + valueMode, known := makeShortOptionArgumentMode(option) + if !known { + return makeArgumentSpec{} + } + if valueMode == makeOptionArgumentNone { + continue + } + return makeArgumentSpec{ + option: true, + valid: true, + valueMode: valueMode, + directory: option == 'C', + valueInline: index+1 < len(argument), + inlineValue: argument[index+1:], + } + } + return makeArgumentSpec{option: true, valid: true} +} + +func malformedShellLineRelevant(line, workingDir string, patterns parserPatterns) bool { + for _, segment := range shellDiagnosticSegments(line) { + arguments, _ := splitMakeCommand(segment) + commandIndex := shellExecutableIndex(arguments) + if commandIndex < len(arguments) && arguments[commandIndex] == "cd" || + isMakeExecutableFromArguments(arguments) || + commandContainsCompiler(segment, arguments, workingDir, patterns) || + compilerBacktickCandidate(segment, workingDir) { + return true + } + } + return false } func splitMakeCommand(line string) ([]string, bool) { @@ -789,33 +637,48 @@ func splitMakeCommand(line string) ([]string, bool) { func makeCommandDirectory(line, workingDir string) (string, bool) { arguments, ok := splitMakeCommand(line) - if !ok || len(arguments) == 0 || !isMakeExecutable(arguments[0]) { + if !ok { + return "", false + } + return makeCommandDirectoryFromArguments(arguments, workingDir) +} + +func makeCommandDirectoryFromArguments(arguments []string, workingDir string) (string, bool) { + commandIndex := shellExecutableIndex(arguments) + if commandIndex >= len(arguments) || !isMakeExecutable(arguments[commandIndex]) { return "", false } directory := workingDir - windowsContext := runtime.GOOS == "windows" || executableBase(arguments[0]) == "mingw32-make" || isExplicitWindowsAbsolutePath(workingDir) + windowsContext := runtime.GOOS == "windows" || executableBase(arguments[commandIndex]) == "mingw32-make" || + isExplicitWindowsAbsolutePath(workingDir) found := false - for i := 1; i < len(arguments); i++ { - argument := arguments[i] - if argument == "--" { + for i := commandIndex + 1; i < len(arguments); i++ { + spec := classifyMakeArgument(arguments[i]) + if !spec.valid { + return "", false + } + if spec.stop { break } - var value string - switch { - case argument == "-C" || argument == "--directory": + if spec.valueMode == makeOptionArgumentNone || + spec.valueMode == makeOptionArgumentOptionalAttached && !spec.valueInline { + continue + } + value := spec.inlineValue + if !spec.valueInline { if i+1 >= len(arguments) { return "", false } i++ value = arguments[i] - case strings.HasPrefix(argument, "-C") && len(argument) > 2: - value = argument[2:] - case strings.HasPrefix(argument, "--directory="): - value = strings.TrimPrefix(argument, "--directory=") - default: + } + if !spec.directory { continue } + if value == "" { + return "", false + } windowsContext = windowsContext || isExplicitWindowsAbsolutePath(value) directory = joinTrackedPathWithWindowsMode(directory, value, windowsContext) found = true @@ -824,13 +687,14 @@ func makeCommandDirectory(line, workingDir string) (string, bool) { } func makeVirtualDirectories(arguments []string, workingDir string) ([]string, bool) { - if len(arguments) == 0 || executableBase(arguments[0]) != "mkdir" { + commandIndex := shellExecutableIndex(arguments) + if commandIndex >= len(arguments) || executableBase(arguments[commandIndex]) != "mkdir" { return nil, false } parents := false directories := []string{} options := true - for _, argument := range arguments[1:] { + for _, argument := range arguments[commandIndex+1:] { if options && argument == "--" { options = false continue @@ -861,13 +725,17 @@ func hasVirtualDirectory(directories map[string]struct{}, directory string) bool return false } -func makeDirectoryEvent(line, event string) (string, bool) { +func makeDirectoryEvent(line, event, makeCommand string) (string, bool) { marker := ": " + event + " directory " index := strings.Index(line, marker) - if index < 0 || !strings.Contains(strings.ToLower(line[:index]), "make") { + if index < 0 || !isMakeDirectoryMarkerPrefix(strings.TrimSpace(line[:index]), makeCommand) { return "", false } value := strings.TrimSpace(line[index+len(marker):]) + return makeDirectoryMarkerValue(value) +} + +func makeDirectoryMarkerValue(value string) (string, bool) { if len(value) < 2 { return "", false } @@ -882,6 +750,27 @@ func makeDirectoryEvent(line, event string) (string, bool) { return value[1 : len(value)-1], true } +func isMakeDirectoryMarkerPrefix(prefix, makeCommand string) bool { + if strings.HasSuffix(prefix, "]") { + open := strings.LastIndexByte(prefix, '[') + if open < 0 || open == len(prefix)-2 { + return false + } + for _, character := range prefix[open+1 : len(prefix)-1] { + if character < '0' || character > '9' { + return false + } + } + prefix = prefix[:open] + } + if strings.Contains(prefix, `\`) && !isRawWindowsAbsolutePathToken(prefix) { + return false + } + base := executableBase(prefix) + return isMakeExecutable(prefix) || strings.HasSuffix(base, "-make") || + makeCommand != "" && base == executableBase(makeCommand) +} + func (t *Tool) expandNestedCommands(line, workingDir string) (string, bool, *shellTokenizationError) { for { expanded, found, ok, parseErr := t.expandNextNestedCommand(line, workingDir) @@ -1566,12 +1455,15 @@ func (t *Tool) Parse(buildLog []string) { } t.Logger.Debug("New command:", line) - // Track make-reported directory changes {{{ - markerLine := "" - if commands, _ := splitShellCommands(line); len(commands) == 1 && commands[0].separator == "" { - markerLine = commands[0].text + commandList, commandListErr := parseShellCommandList(line) + makeCommand := t.Config.MakeCommand + if makeCommand == "" { + makeCommand = makePath } - if directory, ok := makeDirectoryEvent(markerLine, "Entering"); ok { + + // Track make-reported directory changes {{{ + markerLine := compatibleMakeDirectoryMarker(commandList, commandListErr, line, makeCommand) + if directory, ok := makeDirectoryEvent(markerLine, "Entering", makeCommand); ok { enterDir := cleanTrackedPath(directory) if len(dirStack) > 0 && dirStack[0].provisional { dirStack[0] = directoryFrame{path: enterDir} @@ -1581,7 +1473,7 @@ func (t *Tool) Parse(buildLog []string) { workingDir = dirStack[0].path t.Logger.Infof("entering change workingDir: %s", workingDir) continue - } else if directory, ok := makeDirectoryEvent(markerLine, "Leaving"); ok { + } else if directory, ok := makeDirectoryEvent(markerLine, "Leaving", makeCommand); ok { leaveDir := cleanTrackedPath(directory) for i := 0; i < len(dirStack)-1; i++ { if cleanTrackedPath(dirStack[i].path) != leaveDir { @@ -1598,35 +1490,56 @@ func (t *Tool) Parse(buildLog []string) { if checkingMake.MatchString(line) { continue } - if hasUnsupportedShellControlStructure(line) { - t.Logger.Debugf("skip unsupported shell control structure: %s", line) + if commandListErr != nil { + if malformedShellLineRelevant(line, workingDir, patterns) { + parseErr := shellLexicalError(line) + if parseErr == nil { + parseErr = shellParseError(commandListErr) + } + t.logTokenizationFailure(lineNumber, workingDir, parseErr) + } + continue + } + if !supportedShellCommandList(commandList, makeCommand) { + t.Logger.Debugf("skip unsupported shell structure: %s", line) continue } lineWorkingDir := workingDir pendingMakeDir := "" pendingMakeSafe := false - previousStatus := shellStatusSuccess - shellCommands, complete := splitShellCommands(line) - if !complete { - t.Logger.Debugf("skip incomplete shell conditional: %s", line) - continue - } - for _, shellCommand := range shellCommands { - execute, known := shellCommandExecution(shellCommand.separator, previousStatus) - if !known { - previousStatus = shellStatusUnknown - pendingMakeSafe = false - continue + pendingMakeGeneration := 0 + summarizeCommand := func(commandText, commandName string, staticName bool) shellCommandSummary { + arguments, ok := splitMakeCommand(commandText) + if !ok { + return shellCommandSummary{statuses: shellStatusEither, changesState: true} } - if !execute { - continue + commandIndex := shellExecutableIndex(arguments) + if commandIndex >= len(arguments) { + return shellCommandSummary{statuses: shellStatusEither, changesState: true} } - commandText := shellCommand.text + status := shellStatusEither + if staticName && (commandName == "true" || commandName == ":") { + status = shellStatusMaySucceed + } else if staticName && commandName == "false" { + status = shellStatusMayFail + } + changesState := !staticName || commandName == "cd" + if !changesState && isMakeExecutable(commandName) { + arguments[commandIndex] = commandName + _, changesState = makeCommandDirectoryFromArguments(arguments, lineWorkingDir) + } + if !changesState && t.makeDirectoryMarkers && executableBase(commandName) == "mkdir" { + arguments[commandIndex] = commandName + _, changesState = makeVirtualDirectories(arguments, lineWorkingDir) + } + return shellCommandSummary{statuses: status, changesState: changesState} + } + processCommand := func(commandText, commandName string, staticName bool) shellCommandResult { originalCommandText := commandText - if hasUnsupportedShellSyntax(commandText) { - previousStatus = shellStatusUnknown - continue + failureResult := func() shellCommandResult { + summary := summarizeCommand(originalCommandText, commandName, staticName) + return shellCommandResult{status: shellStatusUnknown, safe: staticName && !summary.changesState} } rawArguments, rawOK := splitMakeCommand(commandText) needsExpansion := false @@ -1637,21 +1550,27 @@ func (t *Tool) Parse(buildLog []string) { } else { needsExpansion = patterns.compile.MatchString(commandText) } - if rawOK && len(rawArguments) > 0 { - needsExpansion = needsExpansion || rawArguments[0] == "cd" || isMakeExecutableFromArguments(rawArguments) || - t.makeDirectoryMarkers && executableBase(rawArguments[0]) == "mkdir" + if rawOK { + commandIndex := shellExecutableIndex(rawArguments) + if commandIndex < len(rawArguments) { + needsExpansion = needsExpansion || staticName && (commandName == "cd" || + isMakeExecutable(commandName) || + t.makeDirectoryMarkers && executableBase(commandName) == "mkdir") + } } if compilerCandidateExpansion { var found bool var ok bool var parseErr *shellTokenizationError commandText, found, ok, parseErr = t.expandNextNestedCommand(commandText, lineWorkingDir) - if !ok || !found { + if !ok { if parseErr != nil { t.logTokenizationFailure(lineNumber, lineWorkingDir, parseErr) } - previousStatus = shellStatusUnknown - continue + return shellCommandResult{status: shellStatusUnknown, safe: true} + } + if !found { + return failureResult() } candidateArguments, parsed := splitMakeCommand(commandText) if !parsed { @@ -1663,13 +1582,17 @@ func (t *Tool) Parse(buildLog []string) { t.splitArgs(commandText, lineNumber, lineWorkingDir) } } - previousStatus = shellStatusUnknown - continue + return failureResult() } if !commandContainsCompiler(commandText, candidateArguments, lineWorkingDir, patterns) { - previousStatus = shellStatusUnknown - continue + return failureResult() + } + candidateIndex := shellExecutableIndex(candidateArguments) + if candidateIndex >= len(candidateArguments) { + return failureResult() } + commandName = candidateArguments[candidateIndex] + staticName = true needsExpansion = true } if needsExpansion && strings.Contains(commandText, "`") { @@ -1680,70 +1603,72 @@ func (t *Tool) Parse(buildLog []string) { if parseErr != nil { t.logTokenizationFailure(lineNumber, lineWorkingDir, parseErr) } - previousStatus = shellStatusUnknown - continue + return failureResult() } } arguments, ok := splitMakeCommand(commandText) if !ok { - if len(arguments) > 0 && (arguments[0] == "cd" || isMakeExecutableFromArguments(arguments) || - commandContainsCompiler(commandText, arguments, lineWorkingDir, patterns)) { + commandIndex := shellExecutableIndex(arguments) + if commandIndex < len(arguments) && arguments[commandIndex] == "cd" || + isMakeExecutableFromArguments(arguments) || + commandContainsCompiler(commandText, arguments, lineWorkingDir, patterns) { t.splitArgs(commandText, lineNumber, lineWorkingDir) } - previousStatus = shellStatusUnknown - continue + return failureResult() + } + commandIndex := shellExecutableIndex(arguments) + if commandIndex >= len(arguments) { + return failureResult() } - if t.makeDirectoryMarkers { + if staticName { + arguments[commandIndex] = commandName + } + commandArguments := arguments[commandIndex:] + if t.makeDirectoryMarkers && staticName && executableBase(commandName) == "mkdir" { if directories, recognized := makeVirtualDirectories(arguments, lineWorkingDir); recognized { for _, directory := range directories { virtualDirectories[cleanTrackedPath(directory)] = struct{}{} } - previousStatus = shellStatusSuccess - continue + return shellCommandResult{status: shellStatusSuccess, safe: true} } } - if ok && len(arguments) > 0 && arguments[0] == "cd" { - if len(arguments) != 2 { - previousStatus = shellStatusUnknown - continue + if staticName && commandName == "cd" { + if len(commandArguments) != 2 { + return shellCommandResult{status: shellStatusUnknown, safe: false} } - nextDir := joinTrackedPath(lineWorkingDir, arguments[1]) + nextDir := joinTrackedPath(lineWorkingDir, commandArguments[1]) if t.Config.NoStrict { lineWorkingDir = nextDir - previousStatus = shellStatusSuccess - continue + return shellCommandResult{status: shellStatusSuccess, safe: true} } info, err := os.Stat(nextDir) virtual := hasVirtualDirectory(virtualDirectories, nextDir) if (err != nil || !info.IsDir()) && !(t.makeDirectoryMarkers && virtual) { - previousStatus = shellStatusFailure - continue + return shellCommandResult{status: shellStatusFailure, safe: true} } lineWorkingDir = nextDir - previousStatus = shellStatusSuccess t.Logger.Infof("Temporarily change workingDir: %s", lineWorkingDir) - continue + return shellCommandResult{status: shellStatusSuccess, safe: true} } - if enterDir, ok := makeCommandDirectory(commandText, lineWorkingDir); ok { + if enterDir, ok := makeCommandDirectoryFromArguments(arguments, lineWorkingDir); staticName && + isMakeExecutable(commandName) && ok { pendingMakeDir = enterDir pendingMakeSafe = true - previousStatus = shellStatusUnknown + pendingMakeGeneration++ } if !commandContainsCompiler(commandText, arguments, lineWorkingDir, patterns) { - if isMakeExecutableFromArguments(arguments) { - previousStatus = shellStatusUnknown - continue + if staticName && isMakeExecutable(commandName) { + return shellCommandResult{status: shellStatusUnknown, safe: true} } - if len(arguments) == 1 && (arguments[0] == "true" || arguments[0] == ":") { - previousStatus = shellStatusSuccess - } else if len(arguments) == 1 && arguments[0] == "false" { - previousStatus = shellStatusFailure - } else { - previousStatus = shellStatusUnknown + if staticName && (commandName == "true" || commandName == ":") { + return shellCommandResult{status: shellStatusSuccess, safe: true} } - continue + if staticName && commandName == "false" { + return shellCommandResult{status: shellStatusFailure, safe: true} + } + return shellCommandResult{status: shellStatusUnknown, safe: staticName} } for _, parsed := range t.processCompileCommand(commandText, lineWorkingDir, lineNumber, patterns) { command := ShellJoinArgs(parsed.arguments) @@ -1755,7 +1680,18 @@ func (t *Tool) Parse(buildLog []string) { t.Logger.Infof("Adding command %d: %s", cmdCnt, command) cmdCnt++ } - previousStatus = shellStatusUnknown + return shellCommandResult{status: shellStatusUnknown, safe: true} + } + statementPendingGeneration := pendingMakeGeneration + lineSafe := evaluateShellCommandList(commandList, line, makeCommand, processCommand, summarizeCommand, func() { + if pendingMakeGeneration != statementPendingGeneration { + pendingMakeSafe = false + } + }, func(shellEvaluation) { + statementPendingGeneration = pendingMakeGeneration + }) + if !lineSafe { + pendingMakeSafe = false } if pendingMakeSafe && pendingMakeDir != "" && !t.makeDirectoryMarkers { dirStack = append([]directoryFrame{{path: pendingMakeDir, provisional: true}}, dirStack...) diff --git a/internal/parser_test.go b/internal/parser_test.go index c7be58d..db66398 100644 --- a/internal/parser_test.go +++ b/internal/parser_test.go @@ -468,6 +468,200 @@ func TestParseHonorsKnownConditionalBranches(t *testing.T) { } } +func TestParseEvaluatesShellASTConservatively(t *testing.T) { + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "false && gcc -c skipped-and.c || gcc -c false-and-or.c", + "true || gcc -c skipped-or.c && gcc -c true-or-and.c", + "unknown-check && gcc -c unknown-and.c", + "unknown-check || gcc -c unknown-or.c", + "unknown-check && gcc -c skipped-unknown.c; gcc -c sequential.c", + "false && gcc -c skipped-nested.c || false || gcc -c nested-fallback.c", + "unknown-check || true && gcc -c absorbed-or.c", + "unknown-check && false || gcc -c absorbed-and.c", + "gcc -c first.c || true && gcc -c after-compiler.c", + "printf '中文'; gcc -c unicode-offset.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + files := make([]string, 0, len(commands)) + for _, command := range commands { + files = append(files, command.File) + } + want := []string{ + "false-and-or.c", + "true-or-and.c", + "sequential.c", + "nested-fallback.c", + "absorbed-or.c", + "absorbed-and.c", + "first.c", + "after-compiler.c", + "unicode-offset.c", + } + if !slices.Equal(files, want) { + t.Fatalf("shell AST branches were evaluated incorrectly:\nwant: %v\ngot: %v", want, files) + } +} + +func TestParseHandlesAssignmentPrefixedCommands(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "MODE=release gcc -c compiler.c", + "MODE=release true ignored && gcc -c true.c", + "MODE=release false ignored || gcc -c false.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 3 { + t.Fatalf("assignment-prefixed commands changed command count: %#v", commands) + } + wantFiles := []string{"compiler.c", "true.c", "false.c"} + for index, want := range wantFiles { + if commands[index].File != want { + t.Fatalf("assignment-prefixed command %d: want %q, got %#v", index, want, commands) + } + } + for _, command := range commands { + if command.Directory != trackedPathToSlash(projectDir) { + t.Fatalf("assignment-prefixed command changed cwd: %#v", commands) + } + } +} + +func TestParseRejectsAssignmentPrefixedTrackedState(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "CDPATH=/ cd tmp; gcc -c cd.c", + "PATH=/nonexistent make -C /forged", + "MODE=release mkdir -p virtual; cd virtual; gcc -c mkdir.c", + "CDPATH=/ :; cd tmp; gcc -c colon.c", + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 1 || commands[0].File != "parent.c" || + commands[0].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("assignment-prefixed tracked state changed cwd: %#v", commands) + } +} + +func TestParseRejectsDynamicTrackedDirectories(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + `cd "$DIR"; gcc -c variable.c`, + `cd ~/sub; gcc -c tilde.c`, + `cd sub*; gcc -c glob.c`, + `cd ""; gcc -c empty.c`, + `cd -; gcc -c previous.c`, + `make -C "$DIR"`, + `make -C sub "$TARGET"`, + `make -C sub *`, + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 1 || commands[0].File != "parent.c" || + commands[0].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("dynamic tracked directory was treated as a literal path: %#v", commands) + } +} + +func TestParseRejectsUnsafeStaticShellAnalysis(t *testing.T) { + projectDir := t.TempDir() + marker := filepath.Join(projectDir, "unsafe-shell-executed") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + nested := "printf marker > " + ShellJoinArgs([]string{marker}) + tool.Parse([]string{ + "e\\xit 0; gcc -I`" + nested + "` -c escaped-exit.c", + "MODE=${COMPILEDB_AST_UNSET:?stop} true; gcc -I`" + nested + "` -c fatal-expansion.c", + `foo\make -C /forged; true`, + "gcc -c parent.c", + }) + + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("unsupported shell line executed backtick: %v", err) + } + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 1 || commands[0].File != "parent.c" || + commands[0].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("unsafe shell analysis changed parser state: %#v", commands) + } +} + +func TestParseStopsAfterUncertainTrackedState(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "unknown-check || cd sub; gcc -c uncertain-cd.c", + "unknown-check || make -C child; gcc -c uncertain-make.c", + "unknown-check || cd sub && false || gcc -c absorbed-after-cd.c", + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 1 || commands[0].File != "parent.c" || + commands[0].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("uncertain tracked state leaked into later commands: %#v", commands) + } +} + func TestParseFailsClosedForComplexShellStructures(t *testing.T) { outputFile := filepath.Join(t.TempDir(), "compile_commands.json") tool := newTestTool(t, Config{ @@ -483,7 +677,18 @@ func TestParseFailsClosedForComplexShellStructures(t *testing.T) { `build() { :; gcc -c function.c; }`, `if false; then :; gcc -c conditional.c; fi`, `case value in value) gcc -c case.c;; esac`, + `while false; do gcc -c while.c; done; gcc -c while-tail.c`, + `for value in one; do gcc -c for.c; done; gcc -c for-tail.c`, + `true | gcc -c pipeline.c; gcc -c pipeline-tail.c`, + `true & gcc -c background.c`, + `! false; gcc -c negated.c`, + `exit 0; gcc -c exit.c`, + `exec true; gcc -c exec.c`, + `eval 'true'; gcc -c eval.c`, + `. ./settings; gcc -c dot.c`, + `source ./settings; gcc -c source.c`, `echo $(printf '); gcc -c substitution.c;')`, + `echo $((1 + 2)); gcc -c arithmetic.c`, `( true;# comment ); gcc -c grouped-comment.c`, `true ># comment; gcc -c redirected-comment.c`, `true 2># comment; gcc -c fd-comment.c`, @@ -496,6 +701,63 @@ func TestParseFailsClosedForComplexShellStructures(t *testing.T) { } } +func TestParseContinuesPastUnrelatedRedirectedCommands(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + MakeCommand: "build-tool", + }) + + tool.Parse([]string{ + `printf 'building' >&2; gcc -c after-prefix.c`, + `gcc -c redirected-compiler.c >/dev/null; gcc -c after-compiler.c`, + `gcc -c before-quoted.c; 'FOO=bar' >/dev/null; gcc -c after-quoted.c`, + `cd sub >/dev/null; gcc -c uncertain-cwd.c`, + `gcc -c before-uncertain.c; cd sub >/dev/null; gcc -c after-uncertain.c`, + `gcc -c before-glob.c; c? sub >/dev/null; gcc -c after-glob.c`, + `gcc -c before-tilde.c; ~tool >/dev/null; gcc -c after-tilde.c`, + `gcc -c before-make.c; make -C sub >/dev/null; gcc -c after-make.c`, + `gcc -c before-configured-make.c; build-tool -C sub >/dev/null; gcc -c after-configured-make.c`, + `gcc -c before-mkdir.c; mkdir -p sub >/dev/null; gcc -c after-mkdir.c`, + `gcc -c before-conditional.c; printf x >/dev/null && gcc -c after-conditional.c`, + `gcc -c before-pipeline.c; printf x >/dev/null | cat; gcc -c after-pipeline.c`, + `gcc -c before-group.c; { printf x >/dev/null; }; gcc -c after-group.c`, + `gcc -c before-dynamic-redir.c; printf x >"$OUTPUT"; gcc -c after-dynamic-redir.c`, + `gcc -c before-fatal-redir.c; printf x >${COMPILEDB_REDIRECT_UNSET:?stop}; gcc -c after-fatal-redir.c`, + `gcc -c before-invalid-fd.c; printf x >¬-a-fd; gcc -c after-invalid-fd.c`, + `gcc -c before-colon.c; : >/definitely/missing/path; gcc -c after-colon.c`, + `gcc -c before-times.c; times >/definitely/missing/path; gcc -c after-times.c`, + `tool\? >/dev/null; gcc -c escaped-command.c`, + `printf x >out\*; gcc -c escaped-redir.c`, + `gcc -c parent.c`, + }) + + commands := readCompilerTestCommands(t, outputFile) + wantFiles := []string{ + "after-prefix.c", + "after-compiler.c", + "before-quoted.c", + "after-quoted.c", + "escaped-command.c", + "escaped-redir.c", + "parent.c", + } + if len(commands) != len(wantFiles) { + t.Fatalf("redirected command changed command count: %#v", commands) + } + for index, want := range wantFiles { + if commands[index].File != want || commands[index].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("redirected command %d: want %q in parent cwd, got %#v", index, want, commands) + } + } +} + func TestParseStopsAtShellComments(t *testing.T) { outputFile := filepath.Join(t.TempDir(), "compile_commands.json") tool := newTestTool(t, Config{ @@ -1401,6 +1663,28 @@ func TestParseRecoversFromCommandFailures(t *testing.T) { } } +func TestParseContinuesAfterDynamicBacktickFailure(t *testing.T) { + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + t.Setenv("PATH", t.TempDir()) + + tool.Parse([]string{ + "`missing-backtick-tool` -c skipped.c; gcc -c valid.c", + "cd `false`; gcc -c uncertain-cwd.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "valid.c" { + t.Fatalf("dynamic backtick failure stopped an independent command: status=%d commands=%#v", tool.StatusCode, commands) + } +} + func TestParseReportsTokenizationFailureWithoutCommandContents(t *testing.T) { outputFile := filepath.Join(t.TempDir(), "compile_commands.json") var logs bytes.Buffer @@ -1449,6 +1733,86 @@ func TestParseReportsTokenizationFailureWithoutCommandContents(t *testing.T) { } } +func TestParseDoesNotExecuteBackticksAfterShellParseError(t *testing.T) { + projectDir := t.TempDir() + marker := filepath.Join(projectDir, "parse-error-executed") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + var logs bytes.Buffer + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.Logger.SetLevel(logrus.ErrorLevel) + tool.Logger.SetOutput(&logs) + + tool.Parse([]string{ + "`printf marker > " + ShellJoinArgs([]string{marker}) + "; printf gcc` -c malformed.c && # incomplete", + "`printf marker > " + ShellJoinArgs([]string{marker}) + "; printf gcc` -c bare-and.c &&", + "`printf marker > " + ShellJoinArgs([]string{marker}) + "; printf gcc` -c bare-or.c ||", + "cc -c valid.c", + }) + + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("AST parse-error path executed backtick: %v", err) + } + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "valid.c" { + t.Fatalf("AST parse error changed parser result: status=%d commands=%#v", tool.StatusCode, commands) + } + if !strings.Contains(logs.String(), "skip malformed command") { + t.Fatalf("missing AST parse-error diagnostic: %q", logs.String()) + } +} + +func TestParseReportsRelevantShellGrammarErrors(t *testing.T) { + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + var logs bytes.Buffer + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.Logger.SetLevel(logrus.ErrorLevel) + tool.Logger.SetOutput(&logs) + + tool.Parse([]string{ + "echo ordinary >", + `echo 'gcc -c quoted.c' >`, + "gcc -DSECRET=value -c malformed.c >", + "cd sub >", + "make -C sub |", + "true; gcc -DSECRET=sequence -c sequence.c >", + "printf x; cd nested >", + "echo x; make -C nested |", + "gcc -c arithmetic.c $((1 SECRET 2))", + "cc -c valid.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if tool.StatusCode != 0 || len(commands) != 1 || commands[0].File != "valid.c" { + t.Fatalf("shell grammar errors changed parser result: status=%d commands=%#v", tool.StatusCode, commands) + } + diagnostic := logs.String() + for _, want := range []string{"build log line 3", "build log line 4", "build log line 5", "build log line 6", + "build log line 7", "build log line 8", "build log line 9", "at byte", "cwd"} { + if !strings.Contains(diagnostic, want) { + t.Fatalf("shell grammar diagnostic lacks %q: %q", want, diagnostic) + } + } + if strings.Count(diagnostic, "skip malformed command") != 7 { + t.Fatalf("unexpected shell grammar diagnostic count: %q", diagnostic) + } + if strings.Contains(diagnostic, "SECRET") || strings.Contains(diagnostic, "ordinary") || strings.Contains(diagnostic, "quoted.c") { + t.Fatalf("shell grammar diagnostic exposed command contents or logged unrelated output: %q", diagnostic) + } +} + func TestParseSupportsKnownCompilerLaunchers(t *testing.T) { outputFile := filepath.Join(t.TempDir(), "compile_commands.json") tool := newTestTool(t, Config{ @@ -1815,12 +2179,34 @@ func TestMakeCommandDirectory(t *testing.T) { "UNC": {line: "make -C sub", base: "//server/share/project", want: "//server/share/project/sub"}, "absolute resets": {line: "make -C one -C /other", base: "/project", want: "/other"}, "option terminator": {line: "make -C one -- -C two", base: "/project", want: "/project/one"}, - "Windows drive": {line: `mingw32-make -C "C:\Program Files\build"`, base: "/project", want: "C:/Program Files/build"}, - "Windows UNC": {line: `make -C "\\server\share\build"`, base: "/project", want: "//server/share/build"}, - "unquoted UNC": {line: `make -C \\server\share\build`, base: "/project", want: "//server/share/build"}, - "relative Windows": {line: `mingw32-make -C sub\dir`, base: "/project", want: "/project/sub/dir"}, - "drive relative": {line: `mingw32-make -C C:sub\dir`, base: "C:/project", want: "C:/project/sub/dir"}, - "POSIX colon": {line: `make -C 1:a`, base: "/project", want: "/project/1:a"}, + "file operand resembles directory": { + line: "make -C sub -f -Cevil", base: "/project", want: "/project/sub", + }, + "include operand resembles directory": { + line: "make -I -Cfake -C real", base: "/project", want: "/project/real", + }, + "terminator consumed as file operand": { + line: "make -C sub -f -- -C two", base: "/project", want: "/project/sub/two", + }, + "long option operand resembles terminator": { + line: "make -C sub --file -- -C two", base: "/project", want: "/project/sub/two", + }, + "attached file operand resembles directory": { + line: "make -C sub -f-Cevil", base: "/project", want: "/project/sub", + }, + "optional short value ends cluster": { + line: "make -C sub -lC/ -j8 -Otarget", base: "/project", want: "/project/sub", + }, + "optional long attached values": { + line: "make --jobs=2 --debug=b --output-sync=target --shuffle=reverse -C sub", + base: "/project", want: "/project/sub", + }, + "Windows drive": {line: `mingw32-make -C "C:\Program Files\build"`, base: "/project", want: "C:/Program Files/build"}, + "Windows UNC": {line: `make -C "\\server\share\build"`, base: "/project", want: "//server/share/build"}, + "unquoted UNC": {line: `make -C \\server\share\build`, base: "/project", want: "//server/share/build"}, + "relative Windows": {line: `mingw32-make -C sub\dir`, base: "/project", want: "/project/sub/dir"}, + "drive relative": {line: `mingw32-make -C C:sub\dir`, base: "C:/project", want: "C:/project/sub/dir"}, + "POSIX colon": {line: `make -C 1:a`, base: "/project", want: "/project/1:a"}, } { t.Run(name, func(t *testing.T) { got, ok := makeCommandDirectory(test.line, test.base) @@ -1831,6 +2217,20 @@ func TestMakeCommandDirectory(t *testing.T) { } } +func TestMakeCommandDirectoryRejectsUnknownOptions(t *testing.T) { + for _, line := range []string{ + "make -xC/forged", + "make -C sub --unknown-option", + "make --always-make=value -C /forged", + } { + t.Run(line, func(t *testing.T) { + if directory, ok := makeCommandDirectory(line, "/project"); ok { + t.Fatalf("invalid Make option established directory %q", directory) + } + }) + } +} + func TestParseDoesNotApplyMakeDirectoryToSiblingCommand(t *testing.T) { for _, separator := range []string{" && ", "; ", " || "} { t.Run(strings.TrimSpace(separator), func(t *testing.T) { @@ -1862,6 +2262,72 @@ func TestParseDoesNotApplyMakeDirectoryToSiblingCommand(t *testing.T) { } } +func TestParseTracksMakeAfterResolvedUnknownCondition(t *testing.T) { + for _, line := range []string{ + "unknown-check || true; make -C sub", + "unknown-check || true && make -C sub", + "make -C sub; unknown-check || true", + } { + t.Run(line, func(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.Parse([]string{line, "gcc -c child.c"}) + + commands := readCompilerTestCommands(t, outputFile) + wantDir := trackedPathToSlash(filepath.Join(projectDir, "sub")) + if len(commands) != 1 || commands[0].File != "child.c" || commands[0].Directory != wantDir { + t.Fatalf("definitely executed Make command did not establish directory frame: %#v", commands) + } + }) + } +} + +func TestParseTracksMakeDirectoryWithDynamicTargets(t *testing.T) { + for _, test := range []struct { + line string + want string + }{ + {line: `make -C sub -- "$TARGET"`, want: "sub"}, + {line: `make -C sub target*`, want: "sub"}, + {line: `make -Csub --directory=child -- "$TARGET"`, want: "sub/child"}, + {line: `make -C sub -f -Cevil`, want: "sub"}, + {line: `make -I -Cfake -C sub`, want: "sub"}, + {line: `make -C sub -f -- -C child`, want: "sub/child"}, + {line: `make -C sub --file -- -C child`, want: "sub/child"}, + {line: `make -C sub -lC/ -j8 -Otarget`, want: "sub"}, + {line: `make --jobs=2 --debug=b --output-sync=target --shuffle=reverse -C sub`, want: "sub"}, + } { + t.Run(test.line, func(t *testing.T) { + projectDir := t.TempDir() + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.Parse([]string{test.line, "gcc -c child.c"}) + + commands := readCompilerTestCommands(t, outputFile) + wantDir := filepath.Join(projectDir, filepath.FromSlash(test.want)) + if len(commands) != 1 || commands[0].File != "child.c" || + commands[0].Directory != trackedPathToSlash(wantDir) { + t.Fatalf("dynamic Make target prevented static directory tracking: %#v", commands) + } + }) + } +} + func TestParseTracksMakeDirectoryAfterInlineCD(t *testing.T) { projectDir := t.TempDir() if err := os.Mkdir(filepath.Join(projectDir, "sub"), 0o755); err != nil { @@ -1982,6 +2448,176 @@ func TestParsePreservesQuotesInsideMakeDirectory(t *testing.T) { } } +func TestParseTracksLiteralCommandSubstitutionInMakeDirectory(t *testing.T) { + projectDir := t.TempDir() + childDir := filepath.Join(projectDir, "obj$(name)") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + `make: Entering directory "` + childDir + `"`, + "gcc -c child.c", + `make: Leaving directory "` + childDir + `"`, + `true; make: Entering directory '/forged'`, + `printf make: Entering directory '/forged-command'`, + `notmake: Entering directory '/forged-name'`, + `make: Entering directory '/forged-partial'; if`, + `make: Entering directory '/forged/garbage' '/..'`, + `not\make: Entering directory '/forged-escape'`, + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 2 || commands[0].Directory != trackedPathToSlash(childDir) || + commands[1].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("literal command substitution in Make marker was not preserved: %#v", commands) + } +} + +func TestMakeDirectoryMarkerValue(t *testing.T) { + for _, test := range []struct { + value string + want string + ok bool + }{ + {value: `'/project/sub'`, want: "/project/sub", ok: true}, + {value: `"/project/sub dir"`, want: "/project/sub dir", ok: true}, + {value: "`/project/sub'", want: "/project/sub", ok: true}, + {value: `'/project/a'b'`, want: `/project/a'b`, ok: true}, + {value: `'/project/a'b c'`, want: `/project/a'b c`, ok: true}, + {value: `'/project/child'"dir'`, want: `/project/child'"dir`, ok: true}, + {value: `/project/sub`}, + {value: `'/project/sub"`}, + } { + t.Run(test.value, func(t *testing.T) { + got, ok := makeDirectoryMarkerValue(test.value) + if got != test.want || ok != test.ok { + t.Fatalf("makeDirectoryMarkerValue(%q) = %q, %t; want %q, %t", + test.value, got, ok, test.want, test.ok) + } + }) + } +} + +func TestParseTracksMakeDirectoryContainingApostropheAndSpace(t *testing.T) { + projectDir := t.TempDir() + childDir := filepath.Join(projectDir, "a'b c") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "make: Entering directory '" + childDir + "'", + "gcc -c child.c", + "make: Leaving directory '" + childDir + "'", + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 2 || commands[0].Directory != trackedPathToSlash(childDir) || + commands[1].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("Make directory marker containing apostrophe and space was not tracked: %#v", commands) + } +} + +func TestMakeDirectoryMarkerPrefix(t *testing.T) { + for _, test := range []struct { + prefix string + makeCommand string + want bool + }{ + {prefix: "make", want: true}, + {prefix: "make[1]", want: true}, + {prefix: "/usr/bin/gmake[12]", want: true}, + {prefix: `C:\\tools\\mingw32-make.exe[2]`, want: true}, + {prefix: "/opt/tools/custom-make[3]", want: true}, + {prefix: "/opt/tools/custom-make[3]", makeCommand: "/opt/tools/custom-make", want: true}, + {prefix: "printf make", want: false}, + {prefix: `not\make`, want: false}, + {prefix: `not\custom-make`, makeCommand: "/opt/tools/custom-make", want: false}, + {prefix: "unrelated", makeCommand: "/opt/tools/custom-make", want: false}, + {prefix: "notmake", want: false}, + {prefix: "make[x]", want: false}, + {prefix: "make[]", want: false}, + } { + t.Run(test.prefix, func(t *testing.T) { + if got := isMakeDirectoryMarkerPrefix(test.prefix, test.makeCommand); got != test.want { + t.Fatalf("isMakeDirectoryMarkerPrefix(%q, %q) = %t, want %t", + test.prefix, test.makeCommand, got, test.want) + } + }) + } +} + +func TestParseTracksUnconfiguredCustomMakeDirectoryMarkers(t *testing.T) { + projectDir := t.TempDir() + childDir := filepath.Join(projectDir, "sub") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + + tool.Parse([]string{ + "custom-make[1]: Entering directory '" + childDir + "'", + "gcc -c child.c", + "custom-make[1]: Leaving directory '" + childDir + "'", + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 2 || commands[0].Directory != trackedPathToSlash(childDir) || + commands[1].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("unconfigured custom Make directory markers were not tracked: %#v", commands) + } +} + +func TestParseTracksConfiguredMakeDirectoryMarkers(t *testing.T) { + projectDir := t.TempDir() + childDir := filepath.Join(projectDir, "sub") + outputFile := filepath.Join(t.TempDir(), "compile_commands.json") + makeCommand := filepath.Join(projectDir, "tools", "custom-make") + tool := newTestTool(t, Config{ + InputFile: "stdin", + OutputFile: outputFile, + BuildDir: projectDir, + MakeCommand: makeCommand, + RegexCompile: RegexCompile, + RegexFile: RegexFile, + NoStrict: true, + }) + tool.Parse([]string{ + "custom-make[1]: Entering directory '" + childDir + "'", + "gcc -c child.c", + "custom-make[1]: Leaving directory '" + childDir + "'", + `not\custom-make[1]: Entering directory '/forged'`, + "gcc -c parent.c", + }) + + commands := readCompilerTestCommands(t, outputFile) + if len(commands) != 2 || commands[0].Directory != trackedPathToSlash(childDir) || + commands[1].Directory != trackedPathToSlash(projectDir) { + t.Fatalf("configured Make directory markers were not tracked: %#v", commands) + } +} + func TestParseConfirmsMakeCommandDirectoryFrame(t *testing.T) { projectDir := t.TempDir() childDir := filepath.Join(projectDir, "sub") diff --git a/internal/shell_parse.go b/internal/shell_parse.go new file mode 100644 index 0000000..5884b25 --- /dev/null +++ b/internal/shell_parse.go @@ -0,0 +1,622 @@ +package internal + +import ( + "errors" + "strings" + + "mvdan.cc/sh/v3/syntax" +) + +type shellStatusSet uint8 + +const ( + shellStatusMaySucceed shellStatusSet = 1 << iota + shellStatusMayFail + shellStatusEither = shellStatusMaySucceed | shellStatusMayFail +) + +type shellCommandSummary struct { + statuses shellStatusSet + changesState bool +} + +type shellCommandResult struct { + status shellCommandStatus + safe bool +} + +type shellEvaluation struct { + statuses shellStatusSet + safe bool +} + +func parseShellCommandList(source string) (*syntax.File, error) { + return syntax.NewParser(syntax.Variant(syntax.LangPOSIX)).Parse(strings.NewReader(source), "") +} + +func supportedShellCommandList(file *syntax.File, makeCommand string) bool { + for _, statement := range file.Stmts { + if !supportedShellStatement(statement) { + if _, ok := redirectedShellStatementSummary(statement, makeCommand); !ok { + return false + } + } + } + return true +} + +func redirectedShellStatementSummary(statement *syntax.Stmt, makeCommand string) (shellEvaluation, bool) { + if statement == nil || statement.Negated || statement.Background || statement.Coprocess || statement.Disown || + len(statement.Redirs) == 0 { + return shellEvaluation{}, false + } + command, ok := statement.Cmd.(*syntax.CallExpr) + if !ok || len(command.Args) == 0 || !supportedShellCall(command) { + return shellEvaluation{}, false + } + name, static := staticShellWord(command.Args[0]) + base := executableBase(name) + configuredMake := makeCommand != "" && base == executableBase(makeCommand) + if !static || !trackableShellPathWord(command.Args[0]) || name == ":" || name == "times" || name == "cd" || + isMakeExecutable(name) || + strings.HasSuffix(base, "-make") || configuredMake || base == "mkdir" { + return shellEvaluation{}, false + } + for _, redirect := range statement.Redirs { + if !supportedShellRedirect(redirect) { + return shellEvaluation{}, false + } + } + return shellEvaluation{statuses: shellStatusEither, safe: true}, true +} + +func supportedShellRedirect(redirect *syntax.Redirect) bool { + if redirect == nil || redirect.Word == nil || redirect.Hdoc != nil { + return false + } + value, static := staticShellWord(redirect.Word) + if !static || !trackableShellPathWord(redirect.Word) { + return false + } + switch redirect.Op { + case syntax.RdrOut, syntax.AppOut, syntax.RdrIn, syntax.RdrInOut, syntax.RdrClob: + return true + case syntax.DplIn, syntax.DplOut: + return value == "0" || value == "1" || value == "2" || value == "-" + default: + return false + } +} + +func supportedShellStatement(statement *syntax.Stmt) bool { + if statement == nil || statement.Negated || statement.Background || statement.Coprocess || statement.Disown || + len(statement.Redirs) != 0 { + return false + } + + switch command := statement.Cmd.(type) { + case *syntax.CallExpr: + return supportedShellCall(command) + case *syntax.BinaryCmd: + return (command.Op == syntax.AndStmt || command.Op == syntax.OrStmt) && + supportedShellStatement(command.X) && supportedShellStatement(command.Y) + default: + return false + } +} + +func supportedShellCall(command *syntax.CallExpr) bool { + if len(command.Args) == 0 { + return false + } + if name, ok := staticShellWord(command.Args[0]); ok { + if shellControlCall(name) { + return false + } + if name == ":" && len(command.Assigns) != 0 { + return false + } + switch { + case name == "cd": + if len(command.Assigns) != 0 || len(command.Args) != 2 || !trackableShellPathWord(command.Args[1]) { + return false + } + if value, static := staticShellWord(command.Args[1]); static && (value == "" || value == "-") { + return false + } + case executableBase(name) == "mkdir": + if len(command.Assigns) != 0 { + return false + } + for _, argument := range command.Args[1:] { + if !trackableShellPathWord(argument) { + return false + } + } + case isMakeExecutable(name): + if len(command.Assigns) != 0 || !trackableMakeDirectoryArguments(command.Args[1:]) { + return false + } + } + } + + supported := true + syntax.Walk(command, func(node syntax.Node) bool { + switch node := node.(type) { + case *syntax.CmdSubst: + if !node.Backquotes { + supported = false + } + return false + case *syntax.ParamExp: + if !simpleShellParameterExpansion(node) { + supported = false + } + return supported + case *syntax.ArithmExp, *syntax.ProcSubst: + supported = false + return false + default: + return supported + } + }) + return supported +} + +func simpleShellParameterExpansion(expansion *syntax.ParamExp) bool { + return expansion.Param != nil && expansion.Flags == nil && !expansion.Excl && !expansion.Length && + !expansion.Width && !expansion.IsSet && expansion.NestedParam == nil && expansion.Index == nil && + len(expansion.Modifiers) == 0 && expansion.Slice == nil && expansion.Repl == nil && + expansion.Names == 0 && expansion.Exp == nil +} + +func trackableShellPathWord(word *syntax.Word) bool { + first := true + var checkParts func([]syntax.WordPart, bool) bool + checkParts = func(parts []syntax.WordPart, quoted bool) bool { + for _, part := range parts { + switch part := part.(type) { + case *syntax.Lit: + for index := 0; index < len(part.Value); index++ { + character := part.Value[index] + if !quoted && character == '\\' && index+1 < len(part.Value) { + index++ + first = false + continue + } + if !quoted && (strings.ContainsRune("*?[", rune(character)) || first && character == '~') { + return false + } + first = false + } + case *syntax.SglQuoted: + first = false + case *syntax.DblQuoted: + if !checkParts(part.Parts, true) { + return false + } + first = false + case *syntax.CmdSubst: + if !part.Backquotes { + return false + } + first = false + default: + return false + } + } + return true + } + return checkParts(word.Parts, false) +} + +func trackableMakeDirectoryArguments(arguments []*syntax.Word) bool { + for i := 0; i < len(arguments); i++ { + argument, static := staticShellWord(arguments[i]) + if !static { + return false + } + spec := classifyMakeArgument(argument) + if !spec.valid { + return false + } + if spec.stop { + return true + } + if !spec.option { + if !safeMakeTargetWord(arguments[i], argument) { + return false + } + continue + } + if spec.valueMode == makeOptionArgumentNone || + spec.valueMode == makeOptionArgumentOptionalAttached && !spec.valueInline { + continue + } + if !spec.valueInline { + i++ + if i >= len(arguments) { + return false + } + if spec.directory { + if !trackableMakeDirectoryWord(arguments[i]) { + return false + } + } else { + value, static := staticShellWord(arguments[i]) + if !static || !safeMakeTargetWord(arguments[i], value) { + return false + } + } + } else if spec.directory { + if !trackableStaticMakeDirectory(spec.inlineValue) { + return false + } + } + } + return true +} + +func safeMakeTargetWord(word *syntax.Word, value string) bool { + if trackableShellPathWord(word) { + return true + } + firstPattern := strings.IndexAny(value, "*?[") + return firstPattern > 0 && value[0] != '-' +} + +func trackableMakeDirectoryWord(word *syntax.Word) bool { + if !trackableShellPathWord(word) { + return false + } + value, static := staticShellWord(word) + return !static || trackableStaticMakeDirectory(value) +} + +func trackableStaticMakeDirectory(value string) bool { + return value != "" && !strings.HasPrefix(value, "~") && !strings.ContainsAny(value, "*?[") +} + +func staticShellWord(word *syntax.Word) (string, bool) { + raw, ok := rawStaticShellWord(word) + if !ok { + return "", false + } + if isRawWindowsAbsolutePathToken(raw) { + return raw, true + } + + var value strings.Builder + var appendParts func([]syntax.WordPart, bool) bool + appendParts = func(parts []syntax.WordPart, quoted bool) bool { + for _, part := range parts { + switch part := part.(type) { + case *syntax.Lit: + for index := 0; index < len(part.Value); index++ { + if part.Value[index] == '\\' && !quoted && index+1 < len(part.Value) { + index++ + } + value.WriteByte(part.Value[index]) + } + case *syntax.SglQuoted: + value.WriteString(part.Value) + case *syntax.DblQuoted: + if !appendParts(part.Parts, true) { + return false + } + default: + return false + } + } + return true + } + if !appendParts(word.Parts, false) { + return "", false + } + return value.String(), true +} + +func rawStaticShellWord(word *syntax.Word) (string, bool) { + var value strings.Builder + var appendParts func([]syntax.WordPart) bool + appendParts = func(parts []syntax.WordPart) bool { + for _, part := range parts { + switch part := part.(type) { + case *syntax.Lit: + value.WriteString(part.Value) + case *syntax.SglQuoted: + value.WriteString(part.Value) + case *syntax.DblQuoted: + if !appendParts(part.Parts) { + return false + } + default: + return false + } + } + return true + } + if !appendParts(word.Parts) { + return "", false + } + return value.String(), true +} + +func shellControlCall(name string) bool { + switch name { + case ".", "break", "builtin", "command", "continue", "eval", "exec", "exit", "export", "read", + "readonly", "return", "set", "shift", "source", "trap", "unset": + return true + default: + return false + } +} + +func evaluateShellCommandList( + file *syntax.File, + source string, + makeCommand string, + process func(string, string, bool) shellCommandResult, + summarize func(string, string, bool) shellCommandSummary, + unknownConditional func(), + statementEvaluated func(shellEvaluation), +) bool { + for _, statement := range file.Stmts { + var result shellEvaluation + if len(statement.Redirs) != 0 { + var ok bool + result, ok = redirectedShellStatementSummary(statement, makeCommand) + if !ok { + return false + } + } else { + result = evaluateShellStatement(statement, source, process, summarize, unknownConditional) + } + statementEvaluated(result) + if !result.safe { + return false + } + } + return true +} + +func evaluateShellStatement( + statement *syntax.Stmt, + source string, + process func(string, string, bool) shellCommandResult, + summarize func(string, string, bool) shellCommandSummary, + unknownConditional func(), +) shellEvaluation { + switch command := statement.Cmd.(type) { + case *syntax.CallExpr: + commandText, ok := shellCommandText(command, source) + if !ok { + return shellEvaluation{statuses: shellStatusEither, safe: false} + } + commandName, staticName := staticShellWord(command.Args[0]) + result := process(commandText, commandName, staticName) + return shellEvaluation{statuses: shellStatusSetFromStatus(result.status), safe: result.safe} + case *syntax.BinaryCmd: + left := evaluateShellStatement(command.X, source, process, summarize, unknownConditional) + if !left.safe { + return left + } + switch command.Op { + case syntax.AndStmt: + if left.statuses == shellStatusMaySucceed { + return evaluateShellStatement(command.Y, source, process, summarize, unknownConditional) + } + if left.statuses == shellStatusMayFail { + return left + } + case syntax.OrStmt: + if left.statuses == shellStatusMayFail { + return evaluateShellStatement(command.Y, source, process, summarize, unknownConditional) + } + if left.statuses == shellStatusMaySucceed { + return left + } + } + unknownConditional() + right := summarizeShellStatement(command.Y, source, summarize) + return shellEvaluation{ + statuses: combineShellStatuses(command.Op, left.statuses, right.statuses), + safe: right.safe, + } + } + return shellEvaluation{statuses: shellStatusEither, safe: false} +} + +func summarizeShellStatement( + statement *syntax.Stmt, + source string, + summarize func(string, string, bool) shellCommandSummary, +) shellEvaluation { + switch command := statement.Cmd.(type) { + case *syntax.CallExpr: + commandText, ok := shellCommandText(command, source) + if !ok { + return shellEvaluation{statuses: shellStatusEither, safe: false} + } + commandName, staticName := staticShellWord(command.Args[0]) + summary := summarize(commandText, commandName, staticName) + if !staticName { + summary.changesState = true + } + if summary.statuses == 0 { + summary.statuses = shellStatusEither + } + return shellEvaluation{statuses: summary.statuses, safe: !summary.changesState} + case *syntax.BinaryCmd: + left := summarizeShellStatement(command.X, source, summarize) + if !left.safe { + return left + } + trigger := shellStatusMaySucceed + if command.Op == syntax.OrStmt { + trigger = shellStatusMayFail + } + if left.statuses&trigger == 0 { + return left + } + right := summarizeShellStatement(command.Y, source, summarize) + return shellEvaluation{ + statuses: combineShellStatuses(command.Op, left.statuses, right.statuses), + safe: right.safe, + } + default: + return shellEvaluation{statuses: shellStatusEither, safe: false} + } +} + +func shellCommandText(command *syntax.CallExpr, source string) (string, bool) { + start := int(command.Pos().Offset()) + end := int(command.End().Offset()) + if start < 0 || end < start || end > len(source) { + return "", false + } + return strings.TrimSpace(source[start:end]), true +} + +func shellStatusSetFromStatus(status shellCommandStatus) shellStatusSet { + switch status { + case shellStatusSuccess: + return shellStatusMaySucceed + case shellStatusFailure: + return shellStatusMayFail + default: + return shellStatusEither + } +} + +func combineShellStatuses(operator syntax.BinCmdOperator, left, right shellStatusSet) shellStatusSet { + var result shellStatusSet + switch operator { + case syntax.AndStmt: + if left&shellStatusMayFail != 0 { + result |= shellStatusMayFail + } + if left&shellStatusMaySucceed != 0 { + result |= right + } + case syntax.OrStmt: + if left&shellStatusMaySucceed != 0 { + result |= shellStatusMaySucceed + } + if left&shellStatusMayFail != 0 { + result |= right + } + } + return result +} + +func shellParseError(parseErr error) *shellTokenizationError { + var syntaxErr syntax.ParseError + if errors.As(parseErr, &syntaxErr) { + return &shellTokenizationError{offset: int(syntaxErr.Pos.Offset()), reason: "invalid shell syntax"} + } + var languageErr syntax.LangError + if errors.As(parseErr, &languageErr) { + return &shellTokenizationError{offset: int(languageErr.Pos.Offset()), reason: "unsupported shell syntax"} + } + return &shellTokenizationError{reason: "invalid shell syntax"} +} + +func shellDiagnosticSegments(source string) []string { + segments := []string{} + start := 0 + var quote byte + escaped := false + backtick := false + flush := func(end int) { + if segment := strings.TrimSpace(source[start:end]); segment != "" { + segments = append(segments, segment) + } + } + for i := 0; i < len(source); i++ { + character := source[i] + if escaped { + escaped = false + continue + } + if quote == '\'' { + if character == quote { + quote = 0 + } + continue + } + if character == '\\' { + escaped = true + continue + } + if backtick { + if character == '`' { + backtick = false + } + continue + } + switch character { + case '\'': + quote = character + case '"': + if quote == character { + quote = 0 + } else if quote == 0 { + quote = character + } + case '`': + backtick = true + case '#': + if quote == 0 && (i == start || i > 0 && (source[i-1] == ' ' || source[i-1] == '\t')) { + flush(i) + return segments + } + case ';', '|', '&': + if quote != 0 { + continue + } + flush(i) + if i+1 < len(source) && source[i+1] == character { + i++ + } + start = i + 1 + } + } + flush(len(source)) + return segments +} + +func compatibleMakeDirectoryMarker(file *syntax.File, parseErr error, source, makeCommand string) string { + if parseErr == nil && !singleMakeDirectoryMarker(file) { + return "" + } + trimmed := strings.TrimSpace(source) + marker := ": Entering directory " + index := strings.Index(trimmed, marker) + if index < 0 { + marker = ": Leaving directory " + index = strings.Index(trimmed, marker) + } + if index < 0 || strings.ContainsAny(trimmed[:index], "#;&|") { + return "" + } + if _, ok := makeDirectoryEvent(trimmed, "Entering", makeCommand); !ok { + if _, ok := makeDirectoryEvent(trimmed, "Leaving", makeCommand); !ok { + return "" + } + } + return trimmed +} + +func singleMakeDirectoryMarker(file *syntax.File) bool { + if file == nil || len(file.Stmts) != 1 { + return false + } + statement := file.Stmts[0] + if statement == nil || statement.Negated || statement.Background || statement.Coprocess || statement.Disown || + len(statement.Redirs) != 0 { + return false + } + command, ok := statement.Cmd.(*syntax.CallExpr) + return ok && len(command.Args) == 4 +} diff --git a/tests/build.log b/tests/build.log index f39a42d..c5b7fdb 100644 --- a/tests/build.log +++ b/tests/build.log @@ -8,7 +8,7 @@ g++ -c test1.cpp \ g++ -c test_none.c ccache-clang-11 -c /opt/compiledb_test/test2.c -o objs/test2.c.o -cd /opt/compiledb_test && printf 't' 1>&2; gcc -c src/test1.c -DNESTED_CMD && +cd /opt/compiledb_test && gcc -c src/test1.c -DNESTED_CMD printf 't' 1>&2; cc -c -DINC=\"t.h\" ../test2.c