diff --git a/builder/build.go b/builder/build.go index 974ddc2a37..8f42a9c8e8 100644 --- a/builder/build.go +++ b/builder/build.go @@ -651,6 +651,11 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe if err != nil { return err } + if strings.HasPrefix(config.Triple(), "wasm") { + if err := compiler.ValidateWasmFunctionParameters(mod); err != nil { + return err + } + } // Make sure stack sizes are loaded from a separate section so they can be // modified after linking. diff --git a/compiler/calls.go b/compiler/calls.go index 257973320e..be11d3aef3 100644 --- a/compiler/calls.go +++ b/compiler/calls.go @@ -102,20 +102,6 @@ func (b *builder) createInvoke(fnType llvm.Type, fn llvm.Value, args []llvm.Valu return b.createCall(fnType, fn, args, name) } -// Expand an argument type to a list that can be used in a function call -// parameter list. -func (c *compilerContext) expandFormalParamType(t llvm.Type, name string, goType types.Type) []paramInfo { - if c.isIndirectAggregate(t) { - return []paramInfo{{ - llvmType: c.dataPtrType, - name: name, - elemSize: c.targetData.TypeAllocSize(t), - flags: paramIsGoParam | paramIsReadonly | paramIsIndirect, - }} - } - return c.expandDirectFormalParamType(t, name, goType) -} - func (c *compilerContext) expandDirectFormalParamType(t llvm.Type, name string, goType types.Type) []paramInfo { switch t.TypeKind() { case llvm.StructTypeKind: @@ -130,34 +116,36 @@ func (c *compilerContext) expandDirectFormalParamType(t llvm.Type, name string, return []paramInfo{c.getParamInfo(t, name, goType)} } -func (c *compilerContext) storedParamType(t llvm.Type, exported bool) llvm.Type { - if c.isIndirectParam(t, exported) { +func (c *compilerContext) storedParamType(t llvm.Type) llvm.Type { + if c.isIndirectAggregate(t) { return c.dataPtrType } return t } -func (c *compilerContext) isIndirectParam(t llvm.Type, exported bool) bool { - return !exported && c.isIndirectAggregate(t) -} - -func (b *builder) appendStoredValueTypes(valueTypes []llvm.Type, values []ssa.Value, exported bool) []llvm.Type { - for _, value := range values { - valueTypes = append(valueTypes, b.storedParamType(b.getLLVMType(value.Type()), exported)) +func (b *builder) appendStoredParamTypes(valueTypes []llvm.Type, params []functionABIParam) []llvm.Type { + for _, param := range params { + if param.indirect { + valueTypes = append(valueTypes, b.dataPtrType) + } else { + valueTypes = append(valueTypes, param.llvmType) + } } return valueTypes } -func (b *builder) appendStoredParamTypes(valueTypes []llvm.Type, params []*types.Var, exported bool) []llvm.Type { - for _, param := range params { - valueTypes = append(valueTypes, b.storedParamType(b.getLLVMType(param.Type()), exported)) +func (b *builder) getCallArguments(values []ssa.Value, params []functionABIParam) []llvm.Value { + args := make([]llvm.Value, len(values)) + for i, value := range values { + args[i] = b.getCallArgument(value, params[i].indirect) } - return valueTypes + return args } func (b *builder) prependIndirectResult(sig *types.Signature, exported bool, params []llvm.Value, name string) []llvm.Value { - if resultType, indirect := b.hasIndirectResult(sig); !exported && indirect { - return append([]llvm.Value{b.createIndirectStorage(resultType, name)}, params...) + abi := b.getFunctionABI(sig, exported) + if abi.indirectResult { + return append([]llvm.Value{b.createIndirectStorage(abi.resultType, name)}, params...) } return params } diff --git a/compiler/compiler.go b/compiler/compiler.go index 51197bac34..4e7797dbb0 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -87,6 +87,7 @@ type compilerContext struct { program *ssa.Program diagnostics []error functionInfos map[*ssa.Function]functionInfo + functionABIs map[functionABIKey]functionABI astComments map[string]*ast.CommentGroup embedGlobals map[string][]*loader.EmbedFile pkg *types.Package @@ -107,6 +108,7 @@ func newCompilerContext(moduleName string, machine llvm.TargetMachine, config *C machine: machine, targetData: machine.CreateTargetData(), functionInfos: map[*ssa.Function]functionInfo{}, + functionABIs: map[functionABIKey]functionABI{}, astComments: map[string]*ast.CommentGroup{}, } @@ -1284,27 +1286,23 @@ func (b *builder) createFunctionStart(intrinsic bool) { } // Load function parameters + abi := b.getFunctionABI(b.fn.Signature, b.info.exported) llvmParamIndex := 0 - if _, indirectResult := b.hasIndirectResult(b.fn.Signature); indirectResult && !b.info.exported { + if abi.indirectResult { b.indirectReturn = b.llvmFn.Param(llvmParamIndex) b.indirectReturn.SetName("return") llvmParamIndex++ } - for _, param := range b.fn.Params { - llvmType := b.getLLVMType(param.Type()) - if b.isIndirectParam(llvmType, b.info.exported) { + for i, param := range b.fn.Params { + llvmType := abi.params[i].llvmType + if abi.params[i].indirect { llvmParam := b.llvmFn.Param(llvmParamIndex) llvmParam.SetName(param.Name()) b.indirectValues[param] = llvmParam llvmParamIndex++ continue } - var paramInfos []paramInfo - if b.info.exported { - paramInfos = b.expandDirectFormalParamType(llvmType, param.Name(), param.Type()) - } else { - paramInfos = b.expandFormalParamType(llvmType, param.Name(), param.Type()) - } + paramInfos := b.expandDirectFormalParamType(llvmType, param.Name(), param.Type()) fields := make([]llvm.Value, 0, 1) for _, info := range paramInfos { param := b.llvmFn.Param(llvmParamIndex) @@ -1444,8 +1442,9 @@ func (b *builder) createFunction() { for _, phi := range b.phis { block := phi.ssa.Block() for i, edge := range phi.ssa.Edges { - llvmVal := b.getCallArgument(edge, false) llvmBlock := b.blockInfo[block.Preds[i].Index].exit + b.SetInsertPointBefore(llvmBlock.LastInstruction()) + llvmVal := b.getCallArgument(edge, b.isIndirectAggregate(b.getLLVMType(edge.Type()))) phi.llvm.AddIncoming([]llvm.Value{llvmVal}, []llvm.BasicBlock{llvmBlock}) } } @@ -1676,9 +1675,8 @@ func (b *builder) getValuePointer(value ssa.Value) llvm.Value { return ptr } -func (b *builder) getCallArgument(value ssa.Value, exported bool) llvm.Value { - paramType := b.getLLVMType(value.Type()) - if b.isIndirectParam(paramType, exported) { +func (b *builder) getCallArgument(value ssa.Value, indirect bool) llvm.Value { + if indirect { return b.getValuePointer(value) } return b.getValue(value, getPos(value)) @@ -2323,18 +2321,21 @@ func (b *builder) createFunctionCall(instr *ssa.CallCommon) (llvm.Value, error) b.createNilCheck(instr.Value, callee, "fpcall") } - var params []llvm.Value - for _, param := range instr.Args { - params = append(params, b.getCallArgument(param, exported)) + abi := b.getFunctionABI(instr.Signature(), exported) + paramOffset := 0 + if instr.IsInvoke() { + abi = b.getInterfaceFunctionABI(instr.Signature()) + paramOffset = 1 } + params := b.getCallArguments(instr.Args, abi.params[paramOffset:]) if instr.IsInvoke() { params = append([]llvm.Value{invokeReceiver}, params...) params = append(params, invokeTypecode) } if !exported { - if resultType, indirectResult := b.hasIndirectResult(instr.Signature()); indirectResult { - result := b.createIndirectStorage(resultType, "call.result") + if abi.indirectResult { + result := b.createIndirectStorage(abi.resultType, "call.result") params = append([]llvm.Value{result}, params...) params = append(params, context) b.createInvoke(calleeType, callee, params, "") @@ -2715,7 +2716,7 @@ func (b *builder) createExpr(expr ssa.Value) (llvm.Value, error) { return b.createMapIteratorNext(rangeVal, llvmRangeVal, it), nil } case *ssa.Phi: - phiType := b.storedParamType(b.getLLVMType(expr.Type()), false) + phiType := b.storedParamType(b.getLLVMType(expr.Type())) phi := b.CreatePHI(phiType, "") b.phis = append(b.phis, phiNode{expr, phi}) return phi, nil diff --git a/compiler/compiler_test.go b/compiler/compiler_test.go index d9188790cd..830a3a0d52 100644 --- a/compiler/compiler_test.go +++ b/compiler/compiler_test.go @@ -174,6 +174,147 @@ func TestOptimizedLargeAggregateABI(t *testing.T) { } } +func TestAggregateFunctionABI(t *testing.T) { + for _, target := range []string{"wasm", "cortex-m-qemu"} { + t.Run(target, func(t *testing.T) { + options := &compileopts.Options{Target: target} + if target != "wasm" { + options.Scheduler = "tasks" + } + mod, errs := testCompilePackage(t, options, "aggregate-abi.go") + if len(errs) != 0 { + for _, err := range errs { + t.Error(err) + } + return + } + defer mod.Dispose() + + passOptions := llvm.NewPassBuilderOptions() + defer passOptions.Dispose() + if err := mod.RunPasses("default", llvm.TargetMachine{}, passOptions); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + + checkFunctionParamABI(t, mod, "main.readDirectAggregates", false, false) + checkFunctionParamABI(t, mod, "main.readLimitAggregates", false, false) + checkFunctionParamABI(t, mod, "main.readBoundaryAggregates", true, false) + checkFunctionParamABI(t, mod, "main.readAggregates", true, false) + checkFunctionParamABI(t, mod, "main.readSingleAggregate", false) + checkFunctionParamABI(t, mod, "main.readThreeAggregates", true, false, false) + checkFunctionParamABI(t, mod, "main.readResultBudget", false, true) + checkFunctionParamABI(t, mod, "readAggregateExport", false) + if target == "wasm" { + if err := ValidateWasmFunctionParameters(mod); err != nil { + t.Error(err) + } + } + }) + } +} + +func checkFunctionParamABI(t *testing.T, mod llvm.Module, name string, indirect ...bool) { + t.Helper() + fn := mod.NamedFunction(name) + if fn.IsNil() { + t.Fatalf("missing function %s", name) + } + paramTypes := fn.GlobalValueType().ParamTypes() + for i, wantIndirect := range indirect { + gotIndirect := paramTypes[i].TypeKind() == llvm.PointerTypeKind + if gotIndirect != wantIndirect { + t.Errorf("%s parameter %d indirect=%t, want %t", name, i, gotIndirect, wantIndirect) + } + } +} + +func TestAggregateExportedInterfaceABI(t *testing.T) { + options := &compileopts.Options{Target: "wasm"} + mod, errs := testCompilePackage(t, options, "aggregate-export-abi.go") + defer mod.Dispose() + + for _, err := range errs { + t.Error(err) + } + + var markedWrappers int + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if attr := fn.GetStringAttributeAtIndex(-1, "tinygo-interface-abi-error"); !attr.IsNil() { + markedWrappers++ + } + } + if markedWrappers != 1 { + t.Errorf("found %d exported interface ABI markers, want 1", markedWrappers) + } + if err := ValidateWasmFunctionParameters(mod); err == nil { + t.Error("missing oversized WebAssembly signature error") + } +} + +func TestValidateWasmFunctionParameters(t *testing.T) { + for _, test := range []struct { + name string + params int + resultFields int + declaration bool + used bool + wantError bool + }{ + {"at limit", 999, 2, false, false, false}, + {"hidden result over limit", 1000, 2, false, false, true}, + {"single result at limit", 1000, 1, false, false, false}, + {"empty result at limit", 1000, 0, false, false, false}, + {"unused declaration", 1001, 0, true, false, false}, + {"used declaration", 1001, 0, true, true, true}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("test") + defer mod.Dispose() + builder := ctx.NewBuilder() + defer builder.Dispose() + + paramTypes := make([]llvm.Type, test.params) + for i := range paramTypes { + paramTypes[i] = ctx.Int32Type() + } + resultFields := make([]llvm.Type, test.resultFields) + for i := range resultFields { + resultFields[i] = ctx.Int32Type() + } + resultType := ctx.StructType(resultFields, false) + fnType := llvm.FunctionType(resultType, paramTypes, false) + fn := llvm.AddFunction(mod, "test", fnType) + if test.declaration { + if test.used { + caller := llvm.AddFunction(mod, "caller", llvm.FunctionType(ctx.VoidType(), nil, false)) + block := ctx.AddBasicBlock(caller, "entry") + builder.SetInsertPointAtEnd(block) + args := make([]llvm.Value, len(paramTypes)) + for i, paramType := range paramTypes { + args[i] = llvm.Undef(paramType) + } + builder.CreateCall(fnType, fn, args, "") + builder.CreateRetVoid() + } + } else { + block := ctx.AddBasicBlock(fn, "entry") + builder.SetInsertPointAtEnd(block) + builder.CreateRet(llvm.Undef(resultType)) + } + + err := ValidateWasmFunctionParameters(mod) + if (err != nil) != test.wantError { + t.Errorf("ValidateWasmFunctionParameters() error = %v, wantError = %t", err, test.wantError) + } + }) + } +} + // normalizeIR canonicalizes LLVM-version-specific IR spellings for comparison // and when regenerating golden files. func normalizeIR(s string) string { @@ -280,6 +421,9 @@ func filterIrrelevantIRLines(lines []string) []string { if strings.HasPrefix(line, "source_filename = ") { continue } + if strings.HasPrefix(line, "@tinygo.indirect-abi = ") { + continue + } if llvmVersion < 15 && strings.HasPrefix(line, "target datalayout = ") { // The datalayout string may vary betewen LLVM versions. // Right now test outputs are for LLVM 15 and higher. diff --git a/compiler/defer.go b/compiler/defer.go index f8078f6b52..35535fbe46 100644 --- a/compiler/defer.go +++ b/compiler/defer.go @@ -413,9 +413,6 @@ func (b *builder) createDefer(instr *ssa.Defer) { next := b.CreateLoad(b.dataPtrType, b.deferPtr, "defer.next") var values llvmValueList - lowerArgument := func(value ssa.Value) llvm.Value { - return b.getCallArgument(value, false) - } if instr.Call.IsInvoke() { // Method call on an interface. @@ -433,7 +430,8 @@ func (b *builder) createDefer(instr *ssa.Defer) { typecode := b.CreateExtractValue(itf, 0, "invoke.func.typecode") receiverValue := b.CreateExtractValue(itf, 1, "invoke.func.receiver") values = newLLVMValueList(callback, next, typecode, receiverValue) - values.appendSSAValues(instr.Call.Args, lowerArgument) + abi := b.getInterfaceFunctionABI(instr.Call.Signature()) + values.append(b.getCallArguments(instr.Call.Args, abi.params[1:])...) } else if callee, ok := instr.Call.Value.(*ssa.Function); ok { // Regular function call. @@ -447,9 +445,8 @@ func (b *builder) createDefer(instr *ssa.Defer) { // runtime._defer fields). values = newLLVMValueList(callback, next) exported := b.getFunctionInfo(callee).exported - values.appendSSAValues(instr.Call.Args, func(value ssa.Value) llvm.Value { - return b.getCallArgument(value, exported) - }) + abi := b.getFunctionABI(callee.Signature, exported) + values.append(b.getCallArguments(instr.Call.Args, abi.params)...) } else if makeClosure, ok := instr.Call.Value.(*ssa.MakeClosure); ok { // Immediately applied function literal with free variables. @@ -473,7 +470,8 @@ func (b *builder) createDefer(instr *ssa.Defer) { // runtime._defer fields, followed by all parameters including the // context pointer). values = newLLVMValueList(callback, next) - values.appendSSAValues(instr.Call.Args, lowerArgument) + abi := b.getFunctionABI(fn.Signature, false) + values.append(b.getCallArguments(instr.Call.Args, abi.params)...) values.append(context) } else if builtin, ok := instr.Call.Value.(*ssa.Builtin); ok { @@ -514,7 +512,8 @@ func (b *builder) createDefer(instr *ssa.Defer) { // runtime._defer fields, followed by all parameters including the // context pointer). values = newLLVMValueList(callback, next, funcValue) - values.appendSSAValues(instr.Call.Args, lowerArgument) + abi := b.getFunctionABI(instr.Call.Signature(), false) + values.append(b.getCallArguments(instr.Call.Args, abi.params)...) } // Make a struct out of the collected values to put in the deferred call @@ -623,7 +622,14 @@ func (b *builder) createRunDefers() { valueTypes = append(valueTypes, b.dataPtrType, b.dataPtrType) } - valueTypes = b.appendStoredValueTypes(valueTypes, callback.Args, false) + abi := b.getFunctionABI(callback.Signature(), false) + params := abi.params + if callback.IsInvoke() { + abi = b.getInterfaceFunctionABI(callback.Signature()) + params = abi.params + params = params[1:] + } + valueTypes = b.appendStoredParamTypes(valueTypes, params) // Extract the params from the struct (including receiver). deferredCallType := b.ctx.StructType(valueTypes, false) @@ -666,7 +672,8 @@ func (b *builder) createRunDefers() { // Get the real defer struct type and cast to it. valueTypes := []llvm.Type{b.uintptrType, b.dataPtrType} exported := b.getFunctionInfo(callback).exported - valueTypes = b.appendStoredParamTypes(valueTypes, getParams(callback.Signature), exported) + abi := b.getFunctionABI(callback.Signature, exported) + valueTypes = b.appendStoredParamTypes(valueTypes, abi.params) deferredCallType := b.ctx.StructType(valueTypes, false) // Extract the params from the struct. @@ -689,7 +696,8 @@ func (b *builder) createRunDefers() { // Get the real defer struct type and cast to it. fn := callback.Fn.(*ssa.Function) valueTypes := []llvm.Type{b.uintptrType, b.dataPtrType} - valueTypes = b.appendStoredParamTypes(valueTypes, getParams(fn.Signature), false) + abi := b.getFunctionABI(fn.Signature, false) + valueTypes = b.appendStoredParamTypes(valueTypes, abi.params) valueTypes = append(valueTypes, b.dataPtrType) // closure deferredCallType := b.ctx.StructType(valueTypes, false) diff --git a/compiler/func.go b/compiler/func.go index 260ff07da0..240836b926 100644 --- a/compiler/func.go +++ b/compiler/func.go @@ -4,17 +4,163 @@ package compiler // in a later step, see func-lowering.go. import ( + "fmt" "go/types" + "sort" "golang.org/x/tools/go/ssa" "tinygo.org/x/go-llvm" ) -// LLVM recursively expands each struct field and array element in parameters -// and results into separate values. It gets very slow with too many values, so -// pass larger aggregates indirectly before LLVM expands them. +// LLVM recursively expands each struct field and array element into separate +// values. Pass larger aggregates indirectly before LLVM expands them. const maxDirectAggregateValues = 1024 +// The WebAssembly JavaScript API limits function types to 1000 parameters. +// Apply the same internal ABI cap on every target. +const maxFunctionParams = 1000 + +type functionABIParam struct { + llvmType llvm.Type + indirect bool + leafCount uint64 +} + +type functionABI struct { + resultType llvm.Type + indirectResult bool + params []functionABIParam +} + +type functionABIKey struct { + signature *types.Signature + exported bool + interfaceReceiver bool + budgetReceiverPtr bool + extraParams uint64 +} + +func (c *compilerContext) getFunctionABI(sig *types.Signature, exported bool) functionABI { + budgetReceiverPtr := sig.Recv() != nil && !exported + extraParams := uint64(0) + if budgetReceiverPtr { + // Keep ordinary parameter decisions identical between concrete method + // calls and interface invokes. + extraParams++ // interface typecode + } + return c.getFunctionABIWithReceiver(sig, exported, false, budgetReceiverPtr, extraParams) +} + +func (c *compilerContext) getInterfaceFunctionABI(sig *types.Signature) functionABI { + return c.getFunctionABIWithReceiver(sig, false, true, false, 1) +} + +// getFunctionABIWithReceiver lowers the fewest aggregate parameters necessary +// to keep the complete scalarized signature within the internal ABI cap. +func (c *compilerContext) getFunctionABIWithReceiver(sig *types.Signature, exported, interfaceReceiver, budgetReceiverPtr bool, extraParams uint64) functionABI { + key := functionABIKey{sig, exported, interfaceReceiver, budgetReceiverPtr, extraParams} + if abi, ok := c.functionABIs[key]; ok { + return abi + } + + abi := functionABI{} + abi.resultType, abi.indirectResult = c.hasIndirectResult(sig) + if exported { + abi.indirectResult = false + } + + for i, param := range getParams(sig) { + llvmType := c.getLLVMType(param.Type()) + if i == 0 && interfaceReceiver { + llvmType = c.dataPtrType + } + leafCount, exceeded := aggregateValueCountLimit(llvmType, 0, maxFunctionParams) + if exceeded { + leafCount = maxFunctionParams + 1 + } + abi.params = append(abi.params, functionABIParam{ + llvmType: llvmType, + indirect: !exported && c.isIndirectAggregate(llvmType), + leafCount: leafCount, + }) + } + + if exported { + c.functionABIs[key] = abi + return abi + } + + count := uint64(1) + extraParams // context and synthetic parameters + if abi.indirectResult { + count++ + } else if aggregateValueCountExceeds(abi.resultType, 1) { + count++ + } + + var candidates []int + for i, param := range abi.params { + if param.indirect { + count++ + continue + } + if i == 0 && budgetReceiverPtr { + count++ + continue + } + count += param.leafCount + switch param.llvmType.TypeKind() { + case llvm.ArrayTypeKind, llvm.StructTypeKind: + if param.leafCount > 1 { + candidates = append(candidates, i) + } + } + } + sort.SliceStable(candidates, func(i, j int) bool { + return abi.params[candidates[i]].leafCount > abi.params[candidates[j]].leafCount + }) + // Minimize ABI changes by lowering the largest aggregates first. + for _, i := range candidates { + if count <= maxFunctionParams { + break + } + abi.params[i].indirect = true + count -= abi.params[i].leafCount - 1 + } + if budgetReceiverPtr && !abi.params[0].indirect { + concreteCount := count - extraParams - 1 + abi.params[0].leafCount + if concreteCount > maxFunctionParams { + abi.params[0].indirect = true + } + } + + c.functionABIs[key] = abi + return abi +} + +// ValidateWasmFunctionParameters checks the final LLVM module before the +// WebAssembly backend expands aggregate parameters into scalar values. +func ValidateWasmFunctionParameters(mod llvm.Module) error { + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if fn.IsDeclaration() && fn.FirstUse().IsNil() { + continue + } + count := uint64(0) + if returnType := fn.GlobalValueType().ReturnType(); returnType.TypeKind() == llvm.ArrayTypeKind || returnType.TypeKind() == llvm.StructTypeKind { + if _, indirect := aggregateValueCountLimit(returnType, 0, 1); indirect { + count++ + } + } + for _, paramType := range fn.GlobalValueType().ParamTypes() { + var exceeded bool + count, exceeded = aggregateValueCountLimit(paramType, count, maxFunctionParams) + if exceeded { + return fmt.Errorf("function %s has more than %d WebAssembly parameters after ABI lowering", fn.Name(), maxFunctionParams) + } + } + } + return nil +} + func (c *compilerContext) getLLVMResultType(sig *types.Signature) llvm.Type { switch sig.Results().Len() { case 0: @@ -36,9 +182,13 @@ func (c *compilerContext) hasIndirectResult(sig *types.Signature) (llvm.Type, bo } func (c *compilerContext) isIndirectAggregate(typ llvm.Type) bool { + return aggregateValueCountExceeds(typ, maxDirectAggregateValues) +} + +func aggregateValueCountExceeds(typ llvm.Type, limit uint64) bool { switch typ.TypeKind() { case llvm.ArrayTypeKind, llvm.StructTypeKind: - _, exceeded := aggregateValueCount(typ, 0) + _, exceeded := aggregateValueCountLimit(typ, 0, limit) return exceeded default: return false @@ -46,24 +196,28 @@ func (c *compilerContext) isIndirectAggregate(typ llvm.Type) bool { } func aggregateValueCount(typ llvm.Type, count uint64) (uint64, bool) { + return aggregateValueCountLimit(typ, count, maxDirectAggregateValues) +} + +func aggregateValueCountLimit(typ llvm.Type, count, limit uint64) (uint64, bool) { switch typ.TypeKind() { case llvm.ArrayTypeKind: length := uint64(typ.ArrayLength()) if length == 0 { return count, false } - elementCount, exceeded := aggregateValueCount(typ.ElementType(), 0) + elementCount, exceeded := aggregateValueCountLimit(typ.ElementType(), 0, limit) if exceeded { return count, true } - if elementCount != 0 && length > (maxDirectAggregateValues-count)/elementCount { + if elementCount != 0 && length > (limit-count)/elementCount { return count, true } return count + length*elementCount, false case llvm.StructTypeKind: for _, field := range typ.StructElementTypes() { var exceeded bool - count, exceeded = aggregateValueCount(field, count) + count, exceeded = aggregateValueCountLimit(field, count, limit) if exceeded { return count, true } @@ -71,7 +225,7 @@ func aggregateValueCount(typ llvm.Type, count uint64) (uint64, bool) { return count, false default: count++ - return count, count > maxDirectAggregateValues + return count, count > limit } } @@ -133,11 +287,17 @@ func (c *compilerContext) getFuncType(typ *types.Signature) llvm.Type { // getLLVMFunctionType returns a LLVM function type for a given signature. func (c *compilerContext) getLLVMFunctionType(typ *types.Signature) llvm.Type { - returnType, indirectResult := c.hasIndirectResult(typ) + var abi functionABI + if typ.Recv() != nil && c.getLLVMType(typ.Recv().Type()).StructName() == "runtime._interface" { + abi = c.getFunctionABIWithReceiver(typ, false, true, false, 0) + } else { + abi = c.getFunctionABI(typ, false) + } + returnType := abi.resultType // Get the parameter types. var paramTypes []llvm.Type - if indirectResult { + if abi.indirectResult { // LLVM expands aggregate returns into scalar leaves before deciding // whether to pass them indirectly, so a large IR return can exhaust // memory. Returning void avoids that expansion and cannot be demoted @@ -145,21 +305,13 @@ func (c *compilerContext) getLLVMFunctionType(typ *types.Signature) llvm.Type { paramTypes = append(paramTypes, c.dataPtrType) returnType = c.ctx.VoidType() } - if typ.Recv() != nil { - recv := c.getLLVMType(typ.Recv().Type()) - if recv.StructName() == "runtime._interface" { - // This is a call on an interface, not a concrete type. - // The receiver is not an interface, but a i8* type. - recv = c.dataPtrType - } - for _, info := range c.expandFormalParamType(recv, "", nil) { - paramTypes = append(paramTypes, info.llvmType) - } - } - for v := range typ.Params().Variables() { - subType := c.getLLVMType(v.Type()) - for _, info := range c.expandFormalParamType(subType, "", nil) { - paramTypes = append(paramTypes, info.llvmType) + for _, param := range abi.params { + if param.indirect { + paramTypes = append(paramTypes, c.dataPtrType) + } else { + for _, info := range c.expandDirectFormalParamType(param.llvmType, "", nil) { + paramTypes = append(paramTypes, info.llvmType) + } } } // All functions take these parameters at the end. diff --git a/compiler/goroutine.go b/compiler/goroutine.go index 8bc7da53cd..4b078efe8f 100644 --- a/compiler/goroutine.go +++ b/compiler/goroutine.go @@ -94,8 +94,14 @@ func (b *builder) createGo(instr *ssa.Go) { prefix = b.getFunctionInfo(b.fn).linkName } - for _, param := range instr.Call.Args { - params = append(params, b.getGoroutineCallArgument(param, exported)...) + abi := b.getFunctionABI(instr.Call.Signature(), exported) + paramOffset := 0 + if instr.Call.IsInvoke() { + abi = b.getInterfaceFunctionABI(instr.Call.Signature()) + paramOffset = 1 + } + for i, param := range instr.Call.Args { + params = append(params, b.getGoroutineCallArgument(param, abi.params[paramOffset+i].indirect)...) } if !context.IsNil() { params = append(params, context) @@ -127,10 +133,10 @@ func (b *builder) createGo(instr *ssa.Go) { b.createCall(fnType, start, []llvm.Value{callee, paramBundle, stackSize, llvm.Undef(b.dataPtrType)}, "") } -func (b *builder) getGoroutineCallArgument(value ssa.Value, exported bool) []llvm.Value { +func (b *builder) getGoroutineCallArgument(value ssa.Value, indirect bool) []llvm.Value { typ := b.getLLVMType(value.Type()) - arg := b.getCallArgument(value, exported) - if b.isIndirectParam(typ, exported) { + arg := b.getCallArgument(value, indirect) + if indirect { return []llvm.Value{b.copyToIndirectStorage(arg, typ, "go.param")} } return b.expandFormalParam(arg) diff --git a/compiler/interface.go b/compiler/interface.go index 84f91cc448..eb996dfdcc 100644 --- a/compiler/interface.go +++ b/compiler/interface.go @@ -1298,11 +1298,58 @@ func (c *compilerContext) getInterfaceInvokeWrapper(fn *ssa.Function, llvmFnType return wrapper } + exported := c.getFunctionInfo(fn).exported + abi := c.getFunctionABI(fn.Signature, exported) + if exported { + internalABI := c.getFunctionABI(fn.Signature, false) + var abiError string + if internalABI.indirectResult { + abiError = fmt.Sprintf("exported method %s with a large aggregate result cannot be called through an interface", fn.RelString(nil)) + } + if abiError == "" { + for _, param := range internalABI.params[1:] { + if param.indirect { + abiError = fmt.Sprintf("exported method %s with an aggregate parameter passed indirectly by the internal ABI cannot be called through an interface", fn.RelString(nil)) + break + } + } + } + if abiError != "" { + resultType := internalABI.resultType + var paramTypes []llvm.Type + if internalABI.indirectResult { + paramTypes = append(paramTypes, c.dataPtrType) + resultType = c.ctx.VoidType() + } + paramTypes = append(paramTypes, c.dataPtrType) + for _, param := range internalABI.params[1:] { + if param.indirect { + paramTypes = append(paramTypes, c.dataPtrType) + continue + } + for _, info := range c.expandDirectFormalParamType(param.llvmType, "", nil) { + paramTypes = append(paramTypes, info.llvmType) + } + } + paramTypes = append(paramTypes, c.dataPtrType) + wrapper = llvm.AddFunction(c.mod, wrapperName, llvm.FunctionType(resultType, paramTypes, false)) + c.addStandardDeclaredAttributes(wrapper) + wrapper.AddFunctionAttr(c.ctx.CreateStringAttribute("tinygo-interface-abi-error", abiError)) + return wrapper + } + } + // Get the expanded receiver type. - receiverType := c.getLLVMType(fn.Signature.Recv().Type()) + receiverType := abi.params[0].llvmType var expandedReceiverType []llvm.Type - receiverIndirect := c.isIndirectAggregate(receiverType) - for _, info := range c.expandFormalParamType(receiverType, "", nil) { + receiverIndirect := abi.params[0].indirect + var receiverInfos []paramInfo + if receiverIndirect { + receiverInfos = []paramInfo{{llvmType: c.dataPtrType}} + } else { + receiverInfos = c.expandDirectFormalParamType(receiverType, "", nil) + } + for _, info := range receiverInfos { expandedReceiverType = append(expandedReceiverType, info.llvmType) } @@ -1317,7 +1364,7 @@ func (c *compilerContext) getInterfaceInvokeWrapper(fn *ssa.Function, llvmFnType // create wrapper function resultOffset := 0 - if _, indirect := c.hasIndirectResult(fn.Signature); indirect { + if abi.indirectResult { resultOffset = 1 } paramTypes := append([]llvm.Type{}, llvmFnType.ParamTypes()[:resultOffset]...) diff --git a/compiler/llvmutil/llvm.go b/compiler/llvmutil/llvm.go index a3edc878d4..e3a63b4959 100644 --- a/compiler/llvmutil/llvm.go +++ b/compiler/llvmutil/llvm.go @@ -228,6 +228,63 @@ func AppendToGlobal(mod llvm.Module, globalName string, values ...llvm.Value) { used.SetLinkage(llvm.AppendingLinkage) } +// RemoveFromGlobal removes values matching the predicate from an appending +// global such as llvm.used. +func RemoveFromGlobal(mod llvm.Module, globalName string, remove func(llvm.Value) bool) { + global := mod.NamedGlobal(globalName) + if global.IsNil() { + return + } + + builder := mod.Context().NewBuilder() + defer builder.Dispose() + initializer := global.Initializer() + var kept []llvm.Value + for i := 0; i < initializer.Type().ArrayLength(); i++ { + value := builder.CreateExtractValue(initializer, i, "") + base := value + for !base.IsAConstantExpr().IsNil() && base.OperandsCount() == 1 { + base = base.Operand(0) + } + if !remove(base) { + kept = append(kept, value) + } + } + global.EraseFromParentAsGlobal() + if len(kept) != 0 { + AppendToGlobal(mod, globalName, kept...) + } +} + +// RemoveGlobalReferences removes one occurrence from targetGlobal for each +// value listed in referenceGlobal, then removes referenceGlobal itself. +func RemoveGlobalReferences(mod llvm.Module, targetGlobal, referenceGlobal string) { + references := mod.NamedGlobal(referenceGlobal) + if references.IsNil() { + return + } + + builder := mod.Context().NewBuilder() + defer builder.Dispose() + initializer := references.Initializer() + values := make(map[llvm.Value]int, initializer.Type().ArrayLength()) + for i := 0; i < initializer.Type().ArrayLength(); i++ { + value := builder.CreateExtractValue(initializer, i, "") + for !value.IsAConstantExpr().IsNil() && value.OperandsCount() == 1 { + value = value.Operand(0) + } + values[value]++ + } + references.EraseFromParentAsGlobal() + RemoveFromGlobal(mod, targetGlobal, func(value llvm.Value) bool { + if values[value] == 0 { + return false + } + values[value]-- + return true + }) +} + // Version returns the LLVM major version. func Version() int { majorStr := strings.Split(llvm.Version, ".")[0] diff --git a/compiler/llvmutil/llvm_test.go b/compiler/llvmutil/llvm_test.go new file mode 100644 index 0000000000..b001f9bec8 --- /dev/null +++ b/compiler/llvmutil/llvm_test.go @@ -0,0 +1,41 @@ +package llvmutil + +import ( + "testing" + + "tinygo.org/x/go-llvm" +) + +func TestRemoveGlobalReferences(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + mod := ctx.NewModule("test") + defer mod.Dispose() + + fnType := llvm.FunctionType(ctx.VoidType(), nil, false) + kept := llvm.AddFunction(mod, "kept", fnType) + kept.SetLinkage(llvm.InternalLinkage) + shared := llvm.AddFunction(mod, "shared", fnType) + shared.SetLinkage(llvm.InternalLinkage) + temporary := llvm.AddFunction(mod, "temporary", fnType) + temporary.SetLinkage(llvm.InternalLinkage) + AppendToGlobal(mod, "llvm.used", kept, shared, shared, temporary) + AppendToGlobal(mod, "temporary.roots", shared, temporary) + + RemoveGlobalReferences(mod, "llvm.used", "temporary.roots") + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + if err := mod.RunPasses("globaldce", llvm.TargetMachine{}, options); err != nil { + t.Fatal(err) + } + + if mod.NamedFunction("kept").IsNil() { + t.Error("permanent root was removed") + } + if mod.NamedFunction("shared").IsNil() { + t.Error("permanent root was removed") + } + if !mod.NamedFunction("temporary").IsNil() { + t.Error("temporary root was retained") + } +} diff --git a/compiler/symbol.go b/compiler/symbol.go index 944f74240e..6df4e84cc7 100644 --- a/compiler/symbol.go +++ b/compiler/symbol.go @@ -79,13 +79,11 @@ func (c *compilerContext) getFunction(fn *ssa.Function) (llvm.Type, llvm.Value) return llvmFn.GlobalValueType(), llvmFn } - retType, indirectResult := c.hasIndirectResult(fn.Signature) - if info.exported { - indirectResult = false - } + abi := c.getFunctionABI(fn.Signature, info.exported) + retType := abi.resultType var paramInfos []paramInfo - if indirectResult { + if abi.indirectResult { paramInfos = append(paramInfos, paramInfo{ llvmType: c.dataPtrType, name: "return", @@ -93,12 +91,16 @@ func (c *compilerContext) getFunction(fn *ssa.Function) (llvm.Type, llvm.Value) }) retType = c.ctx.VoidType() } - for _, param := range getParams(fn.Signature) { - paramType := c.getLLVMType(param.Type()) - if info.exported { - paramInfos = append(paramInfos, c.expandDirectFormalParamType(paramType, param.Name(), param.Type())...) + for i, param := range getParams(fn.Signature) { + if abi.params[i].indirect { + paramInfos = append(paramInfos, paramInfo{ + llvmType: c.dataPtrType, + name: param.Name(), + elemSize: c.targetData.TypeAllocSize(abi.params[i].llvmType), + flags: paramIsGoParam | paramIsReadonly | paramIsIndirect, + }) } else { - paramInfos = append(paramInfos, c.expandFormalParamType(paramType, param.Name(), param.Type())...) + paramInfos = append(paramInfos, c.expandDirectFormalParamType(abi.params[i].llvmType, param.Name(), param.Type())...) } } @@ -109,7 +111,7 @@ func (c *compilerContext) getFunction(fn *ssa.Function) (llvm.Type, llvm.Value) } var paramTypes []llvm.Type - hasIndirectABI := indirectResult + hasIndirectABI := abi.indirectResult for _, info := range paramInfos { paramTypes = append(paramTypes, info.llvmType) hasIndirectABI = hasIndirectABI || info.flags¶mIsIndirect != 0 @@ -120,8 +122,10 @@ func (c *compilerContext) getFunction(fn *ssa.Function) (llvm.Type, llvm.Value) if hasIndirectABI { // Argument promotion only rewrites functions whose uses are all direct // calls. Keep an address use so LLVM cannot reconstruct the large - // aggregate signature that this ABI exists to avoid. + // aggregate signature that this ABI exists to avoid. The optimizer + // removes this temporary root before its final dead-code elimination. llvmutil.AppendToGlobal(c.mod, "llvm.used", llvmFn) + llvmutil.AppendToGlobal(c.mod, "tinygo.indirect-abi", llvmFn) } if strings.HasPrefix(c.Triple, "wasm") { // C functions without prototypes like this: diff --git a/compiler/testdata/aggregate-abi.go b/compiler/testdata/aggregate-abi.go new file mode 100644 index 0000000000..5f1cd57a35 --- /dev/null +++ b/compiler/testdata/aggregate-abi.go @@ -0,0 +1,119 @@ +package main + +type aggregateValue [600]int32 + +type directAggregate [499]int32 + +type boundaryAggregate [500]int32 + +type moderateAggregate [400]int32 + +type interfaceBoundaryAggregate [998]int32 + +//go:noinline +func readDirectAggregates(x, y directAggregate) int32 { + return x[0] + y[len(y)-1] +} + +//go:noinline +func readLimitAggregates(x, y directAggregate) (int32, int32) { + return x[0], y[0] +} + +//go:noinline +func readBoundaryAggregates(x, y boundaryAggregate) int32 { + return x[0] + y[len(y)-1] +} + +//go:noinline +func readAggregates(x, y aggregateValue) int32 { + return x[0] + y[len(y)-1] +} + +//go:noinline +func readSingleAggregate(value aggregateValue) int32 { + return value[0] +} + +//go:noinline +func readThreeAggregates(x, y, z moderateAggregate) int32 { + return x[0] + y[0] + z[0] +} + +//go:noinline +func readResultBudget(x directAggregate, y boundaryAggregate) (int32, int32) { + return x[0], y[0] +} + +func selectAggregate(cond bool, x, y aggregateValue) int32 { + selected := x + if cond { + selected = y + } + return readSingleAggregate(selected) +} + +func callAggregates(x, y aggregateValue) int32 { + return readAggregates(x, y) +} + +func callAggregateFunction(fn func(aggregateValue, aggregateValue) int32, x, y aggregateValue) int32 { + return fn(x, y) +} + +type aggregateReceiver struct{} + +type aggregateInterface interface { + read(aggregateValue, aggregateValue) int32 +} + +type boundaryInterface interface { + readBoundary(interfaceBoundaryAggregate) int32 +} + +//go:noinline +func (aggregateReceiver) read(x, y aggregateValue) int32 { + return x[0] + y[len(y)-1] +} + +func callAggregateMethod(receiver aggregateReceiver, x, y aggregateValue) int32 { + return receiver.read(x, y) +} + +func callBoundAggregateMethod(receiver aggregateReceiver, x, y aggregateValue) int32 { + method := receiver.read + return method(x, y) +} + +func callAggregateInterface(receiver aggregateInterface, x, y aggregateValue) int32 { + return receiver.read(x, y) +} + +func (aggregateReceiver) readBoundary(value interfaceBoundaryAggregate) int32 { + return value[0] +} + +func callBoundaryInterface(receiver boundaryInterface, value interfaceBoundaryAggregate) int32 { + return receiver.readBoundary(value) +} + +func deferAggregates(x, y aggregateValue) { + defer readAggregates(x, y) +} + +func deferAggregateFunction(fn func(aggregateValue, aggregateValue) int32, x, y aggregateValue) { + defer fn(x, y) +} + +func goAggregates(x, y aggregateValue) { + go readAggregates(x, y) +} + +func goAggregateFunction(fn func(aggregateValue, aggregateValue) int32, x, y aggregateValue) { + go fn(x, y) +} + +//export readAggregateExport +func readAggregateExport(value aggregateValue) int32 { + return value[0] +} diff --git a/compiler/testdata/aggregate-export-abi.go b/compiler/testdata/aggregate-export-abi.go new file mode 100644 index 0000000000..6675e837a3 --- /dev/null +++ b/compiler/testdata/aggregate-export-abi.go @@ -0,0 +1,31 @@ +package main + +type exportedAggregateParamMethod struct{} + +//export exportedAggregateParamMethodCall +func (exportedAggregateParamMethod) call(value [600]int32, other [600]int32) int32 { + return value[0] + other[len(other)-1] +} + +//export exportedOversizedAggregate +func exportedOversizedAggregate(value [1001]int32) { +} + +type exportedLargeReceiver [600]int32 + +//export exportedLargeReceiverCall +func (receiver exportedLargeReceiver) call(value [399]int32) int32 { + return receiver[0] + value[0] +} + +func exerciseExportedAggregateMethods() { + var paramMethod interface { + call([600]int32, [600]int32) int32 + } = exportedAggregateParamMethod{} + paramMethod.call([600]int32{}, [600]int32{}) + + var receiverMethod interface { + call([399]int32) int32 + } = exportedLargeReceiver{} + receiverMethod.call([399]int32{}) +} diff --git a/transform/interface-lowering.go b/transform/interface-lowering.go index 9c1adf7247..7233a8dcb3 100644 --- a/transform/interface-lowering.go +++ b/transform/interface-lowering.go @@ -29,6 +29,7 @@ package transform // compiler does it: https://research.swtch.com/interfaces import ( + "fmt" "sort" "strings" @@ -275,6 +276,12 @@ func (p *lowerInterfacesPass) run() error { invokeAttr := fn.GetStringAttributeAtIndex(-1, "tinygo-invoke") itf := p.interfaces[methodsAttr.GetStringValue()] signature := itf.signatures[invokeAttr.GetStringValue()] + for _, typ := range itf.types { + function := typ.getMethod(signature).function + if attr := function.GetStringAttributeAtIndex(-1, "tinygo-interface-abi-error"); !attr.IsNil() { + return fmt.Errorf("%s", attr.GetStringValue()) + } + } p.defineInterfaceMethodFunc(fn, itf, signature) } diff --git a/transform/optimizer.go b/transform/optimizer.go index 150a9a77cb..5722678ced 100644 --- a/transform/optimizer.go +++ b/transform/optimizer.go @@ -126,17 +126,6 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { } } - if config.Scheduler() == "none" { - // Check for any goroutine starts. - if start := mod.NamedFunction("internal/task.start"); !start.IsNil() && len(getUses(start)) > 0 { - errs := []error{} - for _, call := range getUses(start) { - errs = append(errs, errorAt(call, "attempted to start a goroutine without a scheduler")) - } - return errs - } - } - if config.VerifyIR() { if errs := ircheck.Module(mod); errs != nil { return errs @@ -167,6 +156,24 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error { return []error{fmt.Errorf("could not build pass pipeline: %w", err)} } + llvmutil.RemoveGlobalReferences(mod, "llvm.used", "tinygo.indirect-abi") + cleanupOptions := llvm.NewPassBuilderOptions() + defer cleanupOptions.Dispose() + if err := mod.RunPasses("globaldce", llvm.TargetMachine{}, cleanupOptions); err != nil { + return []error{fmt.Errorf("could not run final globaldce pass: %w", err)} + } + + if config.Scheduler() == "none" { + // Check for any goroutine starts. + if start := mod.NamedFunction("internal/task.start"); !start.IsNil() && len(getUses(start)) > 0 { + errs := []error{} + for _, call := range getUses(start) { + errs = append(errs, errorAt(call, "attempted to start a goroutine without a scheduler")) + } + return errs + } + } + hasGCPass := MakeGCStackSlots(mod) if hasGCPass { if err := llvm.VerifyModule(mod, llvm.PrintMessageAction); err != nil {