From 1a4a26b78baf6aade8d12b2b71023adee1a93336 Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Tue, 30 Jun 2026 08:45:52 +0200 Subject: [PATCH 1/6] feat: QONNX parsing and basic lowering --- core/CMakeLists.txt | 5 + core/inc/SOFIE/OperatorList.hxx | 1 + core/inc/SOFIE/RModel.hxx | 29 ++ core/inc/SOFIE/ROperator.hxx | 7 +- core/inc/SOFIE/ROperator_Gemm.hxx | 15 + core/inc/SOFIE/ROperator_QONNXQuant.hxx | 163 ++++++ core/inc/SOFIE/ROperator_QuantizedGemm.hxx | 418 ++++++++++++++++ core/inc/SOFIE/RQuantization.hxx | 225 +++++++++ core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx | 268 ++++++++++ core/inc/SOFIE/SOFIE_common.hxx | 1 + core/src/RModel.cxx | 24 +- core/src/RModel_Quantization.cxx | 556 +++++++++++++++++++++ parsers/CMakeLists.txt | 1 + parsers/inc/SOFIE/RModelParser_ONNX.hxx | 14 +- parsers/src/ParseQONNXQuant.cxx | 88 ++++ parsers/src/RModelParser_ONNX.cxx | 112 ++++- 16 files changed, 1908 insertions(+), 19 deletions(-) create mode 100644 core/inc/SOFIE/ROperator_QONNXQuant.hxx create mode 100644 core/inc/SOFIE/ROperator_QuantizedGemm.hxx create mode 100644 core/inc/SOFIE/RQuantization.hxx create mode 100644 core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx create mode 100644 core/src/RModel_Quantization.cxx create mode 100644 parsers/src/ParseQONNXQuant.cxx diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index a99f6d4..99d3d86 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -27,6 +27,9 @@ set(sources_headers SOFIE/ROperator_Conv.hxx SOFIE/ROperator_ConvTranspose.hxx SOFIE/ROperator_Gemm.hxx + SOFIE/ROperator_QuantizedGemm.hxx + SOFIE/SOFIE_QuantizedRuntime.hxx + SOFIE/RQuantization.hxx SOFIE/ROperator_Relu.hxx SOFIE/ROperator_Tanh.hxx SOFIE/ROperator_LeakyRelu.hxx @@ -57,6 +60,7 @@ set(sources_headers SOFIE/ROperator_Split.hxx SOFIE/ROperator_SubGraph.hxx SOFIE/ROperator_Pad.hxx + SOFIE/ROperator_QONNXQuant.hxx SOFIE/ROperator_Where.hxx SOFIE/ROperator_Einsum.hxx SOFIE/ROperator_Random.hxx @@ -78,6 +82,7 @@ list(TRANSFORM sources_headers PREPEND "inc/") set(sources_cxx src/RModel_Base.cxx src/RModel.cxx + src/RModel_Quantization.cxx src/RModelProfiler.cxx src/RModelProfilerGPU.cxx src/RModel_ALPAKA.cxx diff --git a/core/inc/SOFIE/OperatorList.hxx b/core/inc/SOFIE/OperatorList.hxx index 02ae60d..03b9185 100644 --- a/core/inc/SOFIE/OperatorList.hxx +++ b/core/inc/SOFIE/OperatorList.hxx @@ -1,5 +1,6 @@ #include "SOFIE/ROperator_Transpose.hxx" #include "SOFIE/ROperator_Gemm.hxx" +#include "SOFIE/ROperator_QuantizedGemm.hxx" #include "SOFIE/ROperator_Relu.hxx" #include "SOFIE/ROperator_Tanh.hxx" #include "SOFIE/ROperator_LeakyRelu.hxx" diff --git a/core/inc/SOFIE/RModel.hxx b/core/inc/SOFIE/RModel.hxx index 8153408..b7943f7 100644 --- a/core/inc/SOFIE/RModel.hxx +++ b/core/inc/SOFIE/RModel.hxx @@ -3,6 +3,7 @@ #include "SOFIE/RModel_Base.hxx" #include "SOFIE/SOFIE_common.hxx" +#include "SOFIE/RQuantization.hxx" #include "SOFIE/ROperator.hxx" @@ -32,6 +33,7 @@ private: std::unordered_map fInitializedTensors; std::unordered_map fIntermediateTensorInfos; std::unordered_map fDynamicTensorInfos; + QuantizationModelState fQuantizationState; // quantization metadata, regions, storage, and backend lowering plans std::unordered_map, bool>> fShapeTensors; // constant tensors describing a shape std::unordered_map fAliasTensors; // alias tensors (name -> original tensor name) std::unordered_map @@ -42,6 +44,11 @@ private: std::vector> fOperators; + // transient lowered operator view while the parsed fOperators graph remains intact. + // let code generation use synthetic operators for proven regions. + std::unordered_map> fLoweredOperators; + std::set fLoweredConsumedOperatorIndices; + std::vector> fSubGraphs; /// GetQuantizedGemmOperatorIndices() const; + bool HasQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const; + const QuantizedLoweringPlan & GetQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const; public: // Rule of five: explicitly define move semantics, disallow copy @@ -100,6 +118,17 @@ public: ETensorType GetTensorType(std::string name) const; std::vector GetDynamicTensorShape(const std::string & name) const ; + void AddQuantizationInfo(const std::string & tensor_name, QuantizationInfo info); + bool HasQuantizationInfo(const std::string & tensor_name) const; + const QuantizationInfo & GetQuantizationInfo(const std::string & tensor_name) const; + + void RegisterQuantizedTensorStorage(QuantizedTensorStorage storage); + bool HasQuantizedTensorStorage(const std::string & storage_tensor_name) const; + const QuantizedTensorStorage & GetQuantizedTensorStorage(const std::string & storage_tensor_name) const; + + void AnalyzeQuantizedRegions(); + const QuantizationModelState & GetQuantizationState() const { return fQuantizationState; } + // get the values for the tensor representing a shape const std::vector & GetShapeTensorValues(const std::string & tensor_name) const; diff --git a/core/inc/SOFIE/ROperator.hxx b/core/inc/SOFIE/ROperator.hxx index c24fd70..c685780 100644 --- a/core/inc/SOFIE/ROperator.hxx +++ b/core/inc/SOFIE/ROperator.hxx @@ -38,12 +38,14 @@ enum class OperatorKind { UNARY_COS=22, UNARY_ABS=23, CLIP=24, - NOT=25 + NOT=25, + QUANTIZED_GEMM=26 }; inline const char* toString(OperatorKind kind) { switch (kind) { case OperatorKind::GEMM: return "GEMM"; + case OperatorKind::QUANTIZED_GEMM: return "QUANTIZED_GEMM"; case OperatorKind::LAYERNORM: return "LAYERNORM"; case OperatorKind::RELU: return "RELU"; case OperatorKind::CONSTANT: return "CONSTANT"; @@ -82,6 +84,9 @@ public: virtual std::string GetBlasConfig() { return ""; } virtual void UpdateFusableTensorName(std::string, const std::function& removal_func){ return;}; + // Semantic graph-analysis hooks + virtual bool IsQuantizationBoundary() const { return false; } + // Elementwise kernel fusion interface virtual bool IsElementwise() const { return false; } // Returns the C++ expression applying this op to inputVar (a local T variable) for fused kernel generation diff --git a/core/inc/SOFIE/ROperator_Gemm.hxx b/core/inc/SOFIE/ROperator_Gemm.hxx index eecb33b..1f58266 100644 --- a/core/inc/SOFIE/ROperator_Gemm.hxx +++ b/core/inc/SOFIE/ROperator_Gemm.hxx @@ -12,6 +12,7 @@ #include #include #include +#include namespace SOFIE{ @@ -72,6 +73,20 @@ namespace SOFIE{ fOutputTensorNames = { fNY }; fKind = OperatorKind::GEMM; } + // Getters for further quantized GEMM lowering-elimination. + float GetAlpha() const { return fAttrAlpha; } + float GetBeta() const { return fAttrBeta; } + int_t GetTransA() const { return fAttrTransA; } + int_t GetTransB() const { return fAttrTransB; } + bool HasBias() const { return !fNC.empty(); } + const std::string &GetInputTensorName() const { return fNA; } + const std::string &GetWeightTensorName() const { return fNB; } + const std::string &GetBiasTensorName() const { return fNC; } + const std::string &GetOutputTensorName() const { return fNY; } + const std::vector &GetInputShape() const { return fShapeA; } + const std::vector &GetWeightShape() const { return fShapeB; } + const std::vector &GetOutputShape() const { return fShapeY; } + std::vector GetStdLibs() override { return { std::string("cmath"), std::string("cstdint"), std::string("vector") }; } std::vector TypeInference(std::vector input) override { ETensorType out = input[0]; diff --git a/core/inc/SOFIE/ROperator_QONNXQuant.hxx b/core/inc/SOFIE/ROperator_QONNXQuant.hxx new file mode 100644 index 0000000..8c61cee --- /dev/null +++ b/core/inc/SOFIE/ROperator_QONNXQuant.hxx @@ -0,0 +1,163 @@ +#ifndef SOFIE_ROPERATOR_QONNXQUANT +#define SOFIE_ROPERATOR_QONNXQUANT + +#include "SOFIE/RModel.hxx" +#include "SOFIE/ROperator.hxx" +#include "SOFIE/SOFIE_common.hxx" +#include "SOFIE/RQuantization.hxx" + +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +// Fallback for qonnx.custom_op.general::Quant. +// Operator implements fake quantization: values are rounded and clamped to +// the quantized integer grid, then converted back to the floating carrier type. +class ROperator_QONNXQuant final : public ROperator { +private: + std::string fNX; + std::string fNScale; + std::string fNZeroPoint; + std::string fNBitWidth; + std::string fNY; + bool fIsSigned = false; + bool fNarrow = false; + EQuantizationRoundingMode fRounding = EQuantizationRoundingMode::UNDEFINED; + EQuantizationOverflowMode fOverflow = EQuantizationOverflowMode::UNDEFINED; + std::vector fShape; + double fScale = 1.0; + std::int64_t fZeroPoint = 0; + unsigned fBitWidth = 0; + double fQMin = 0.0; + double fQMax = 0.0; + + static float GetScalarFloat(RModel &model, const std::string &tensorName) + { + auto values = model.GetTensorData(tensorName); + if (values.size() != 1) { + throw std::runtime_error("SOFIE QONNX Quant expected scalar FLOAT initializer " + tensorName); + } + return values.front(); + } + + std::string RoundingExpression(const std::string &value) const + { + switch (fRounding) { + case EQuantizationRoundingMode::ROUND: return "std::nearbyint(" + value + ")"; + case EQuantizationRoundingMode::FLOOR: return "std::floor(" + value + ")"; + case EQuantizationRoundingMode::TRUNCATE: return "std::trunc(" + value + ")"; + default: throw std::runtime_error("SOFIE QONNX Quant has unsupported rounding mode"); + } + } + +public: + ROperator_QONNXQuant() = default; + + ROperator_QONNXQuant(std::string nameX, std::string nameScale, std::string nameZeroPoint, + std::string nameBitWidth, std::string nameY, bool isSigned, bool narrow, + EQuantizationRoundingMode rounding, EQuantizationOverflowMode overflow) + : fNX(UTILITY::Clean_name(nameX)), fNScale(UTILITY::Clean_name(nameScale)), + fNZeroPoint(UTILITY::Clean_name(nameZeroPoint)), fNBitWidth(UTILITY::Clean_name(nameBitWidth)), + fNY(UTILITY::Clean_name(nameY)), fIsSigned(isSigned), fNarrow(narrow), fRounding(rounding), + fOverflow(overflow) + { + fInputTensorNames = { fNX, fNScale, fNZeroPoint, fNBitWidth }; + fOutputTensorNames = { fNY }; + } + + bool IsQuantizationBoundary() const override { return true; } + + std::vector TypeInference(std::vector input) override + { + if (input.empty()) return {}; + return { input.front() }; + } + + std::vector> ShapeInference(std::vector> input) override + { + if (input.empty()) return {}; + return { input.front() }; + } + + void Initialize(RModel &model) override + { + if (!model.CheckIfTensorAlreadyExist(fNX)) { + throw std::runtime_error("SOFIE QONNX Quant input tensor " + fNX + " is not found in model"); + } + if (!model.IsInitializedTensor(fNScale) || !model.IsInitializedTensor(fNZeroPoint) || + !model.IsInitializedTensor(fNBitWidth)) { + throw std::runtime_error("SOFIE QONNX Quant scale, zero-point, and bit-width must be initialized tensors"); + } + + fScale = static_cast(GetScalarFloat(model, fNScale)); + const double zeroPointFloat = static_cast(GetScalarFloat(model, fNZeroPoint)); + const double bitWidthFloat = static_cast(GetScalarFloat(model, fNBitWidth)); + + if (!(fScale > 0.0)) { + throw std::runtime_error("SOFIE QONNX Quant scale must be positive for tensor " + fNY); + } + if (std::round(zeroPointFloat) != zeroPointFloat) { + throw std::runtime_error("SOFIE QONNX Quant zero-point must be integral for tensor " + fNY); + } + if (std::round(bitWidthFloat) != bitWidthFloat || bitWidthFloat <= 0.0) { + throw std::runtime_error("SOFIE QONNX Quant bit-width must be a positive integer for tensor " + fNY); + } + + fZeroPoint = static_cast(zeroPointFloat); + fBitWidth = static_cast(bitWidthFloat); + + QuantizationInfo info; + info.bitWidth = fBitWidth; + info.isSigned = fIsSigned; + info.narrow = fNarrow; + info.scale = fScale; + info.zeroPoint = fZeroPoint; + info.rounding = fRounding; + info.overflow = fOverflow; + info.granularity = EQuantizationGranularity::PerTensor; + info.axis = -1; + + auto [qMin, qMax] = QuantizedIntegerRange(info); + fQMin = static_cast(qMin); + fQMax = static_cast(qMax); + model.AddQuantizationInfo(fNY, std::move(info)); + + fShape = model.GetTensorShape(fNX); + model.AddIntermediateTensor(fNY, model.GetTensorType(fNX), fShape); + model.AddNeededStdLib("cmath"); + } + + std::string Generate(std::string OpName) override + { + OpName = "op_" + OpName; + if (fBitWidth == 0) { + throw std::runtime_error("SOFIE QONNX Quant called to Generate without being initialized first"); + } + const auto length = ConvertShapeToLength(fShape); + const std::string scaledValue = "((static_cast(tensor_" + fNX + "[id]) / " + std::to_string(fScale) + ") + " + std::to_string(fZeroPoint) + ")"; + + std::stringstream out; + out << "\n//------ QONNX QUANT FAKE-QUANT " << OpName << "\n"; + out << SP << "for (size_t id = 0; id < " << length << "; ++id) {\n"; + out << SP << SP << "double q = " << RoundingExpression(scaledValue) << ";\n"; + out << SP << SP << "q = (q < " << fQMin << ") ? " << fQMin << " : ((q > " << fQMax << ") ? " << fQMax << " : q);\n"; + out << SP << SP << "tensor_" << fNY << "[id] = (q - " << fZeroPoint << ") * " << fScale << ";\n"; + out << SP << "}\n"; + return out.str(); + } + + std::string Generate_GPU_ALPAKA(std::string /*OpName*/) override + { + throw std::runtime_error("SOFIE QONNX Quant Alpaka code generation is not available"); + } +}; + +} // namespace SOFIE + +#endif // SOFIE_ROPERATOR_QONNXQUANT diff --git a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx new file mode 100644 index 0000000..9c5103a --- /dev/null +++ b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx @@ -0,0 +1,418 @@ +#ifndef SOFIE_ROPERATOR_QUANTIZED_GEMM +#define SOFIE_ROPERATOR_QUANTIZED_GEMM + +#include "SOFIE/ROperator.hxx" +#include "SOFIE/SOFIE_common.hxx" +#include "SOFIE/RQuantization.hxx" +#include "SOFIE/SOFIE_QuantizedRuntime.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +struct QuantizedGemmCodegenContext { + std::string inputTensor; + std::string weightTensor; + std::string biasTensor; + std::string outputTensor; + std::vector inputShape; + std::vector weightShape; + std::vector outputShape; + float alpha = 1.0f; + float beta = 1.0f; + std::int64_t transA = 0; + std::int64_t transB = 0; + EActivationType activation = EActivationType::UNDEFINED; + std::string indent = " "; +}; + +namespace INTERNAL { + +inline void ValidateQuantizedGemmContext(const QuantizedGemmCodegenContext &context, + const std::string &pathName) +{ + if (context.inputShape.empty() || context.weightShape.empty() || context.outputShape.empty()) { + throw std::runtime_error("SOFIE " + pathName + " called before Gemm initialization"); + } + if (context.transA != 0 || context.transB != 1) { + throw std::runtime_error("SOFIE " + pathName + " supports transA=0 and transB=1"); + } + if (context.alpha != 1.0f || context.beta != 1.0f) { + throw std::runtime_error("SOFIE " + pathName + " supports alpha=1 and beta=1"); + } + if (context.activation != EActivationType::UNDEFINED && context.activation != EActivationType::RELU) { + throw std::runtime_error("SOFIE " + pathName + " supports only no fused activation or fused ReLU"); + } + if (context.inputShape.size() != 2 || context.weightShape.size() != 2 || context.outputShape.size() != 2) { + throw std::runtime_error("SOFIE " + pathName + " supports rank-2 Gemm only"); + } +} + +inline bool NearlyEqualQuantizedScale(double lhs, double rhs) +{ + const auto scale = std::max({1.0, std::fabs(lhs), std::fabs(rhs)}); + return std::fabs(lhs - rhs) <= (1.0e-12 * scale); +} + +inline bool MakeQuantizedGemmFixedPointMultiplier(double realMultiplier, std::int64_t &multiplier, int &shift) +{ + if (!std::isfinite(realMultiplier) || realMultiplier <= 0.0) { + return false; + } + + int exponent = 0; + const double significand = std::frexp(realMultiplier, &exponent); + auto q31 = static_cast(std::llround(significand * 2147483648.0)); + if (q31 == (std::int64_t{1} << 31)) { + q31 /= 2; + ++exponent; + } + + const int candidateShift = 31 - exponent; + if (q31 <= 0 || candidateShift < 0 || candidateShift >= 62) { + return false; + } + + multiplier = q31; + shift = candidateShift; + return true; +} + +inline bool MakeExactIntegerScaleMultiplier(double scale, std::int64_t &multiplier, int &shift) +{ + if (!std::isfinite(scale) || scale < 0.0) { + return false; + } + + const auto rounded = std::llround(scale); + if (!NearlyEqualQuantizedScale(scale, static_cast(rounded))) { + return false; + } + + multiplier = rounded; + shift = 0; + return true; +} + +} // namespace INTERNAL + +inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, const QuantizedGemmCodegenContext &context, + const QuantizedGemmRegion ®ion, + const QuantizedLoweringPlan &plan) +{ + opName = "op_" + opName; + INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm call CPU"); + + if (!plan.usesPrequantizedWeights) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path requires pre-quantized weight storage"); + } + if (plan.weightStorageTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path is missing a weight storage tensor"); + } + if (plan.weightLayout != EQuantizedLayout::PackedCPU) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path requires PackedCPU weight layout"); + } + if (plan.weightStorage != EQuantizedStorageType::Int8 && plan.weightStorage != EQuantizedStorageType::UInt8) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path requires int8/uint8 weight storage"); + } + if (region.inputSourceTensor.empty() || region.outputTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path is missing quantization source/output tensors"); + } + if (!context.biasTensor.empty() && (region.biasSourceTensor.empty() || !region.biasQuant.has_value())) { + throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path is missing quantized bias metadata"); + } + + const auto dimA = context.inputShape.size(); + const auto dimB = context.weightShape.size(); + const auto m = context.inputShape[dimA - 2].GetVal(); + const auto k = context.inputShape[dimA - 1].GetVal(); + const auto n = context.weightShape[dimB - 2].GetVal(); + const auto &SP = context.indent; + constexpr std::size_t tileN = 4; + + const auto inputRange = QuantizedIntegerRange(region.inputQuant); + const auto outputRange = QuantizedIntegerRange(region.outputQuant); + + std::stringstream out; + out << "\n//--------- Fused Quantized Gemm call CPU " << opName << " " + << ConvertDimShapeToString(context.inputShape) << " * " << ConvertDimShapeToString(context.weightShape) + << " -> " << ConvertDimShapeToString(context.outputShape) << "\n"; + out << SP << "// Call-boundary path: generated code delegates quantized GEMM to SOFIE::QuantizedGemm_Call(...).\n"; + out << SP << "// The runtime call consumes quantized parameters and packed constant weights.\n"; + + out << SP << "SOFIE::QuantizedGemmParams " << opName << "_params{};\n"; + out << SP << opName << "_params.m = static_cast(" << m << ");\n"; + out << SP << opName << "_params.n = static_cast(" << n << ");\n"; + out << SP << opName << "_params.k = static_cast(" << k << ");\n"; + out << SP << opName << "_params.tileN = " << tileN << ";\n"; + + out << SP << opName << "_params.scaleX = " + << std::setprecision(std::numeric_limits::max_digits10) << region.inputQuant.scale << ";\n"; + out << SP << opName << "_params.scaleW = " + << std::setprecision(std::numeric_limits::max_digits10) << region.weightQuant.scale << ";\n"; + out << SP << opName << "_params.scaleY = " + << std::setprecision(std::numeric_limits::max_digits10) << region.outputQuant.scale << ";\n"; + out << SP << opName << "_params.zeroX = " << region.inputQuant.zeroPoint << ";\n"; + out << SP << opName << "_params.zeroW = " << region.weightQuant.zeroPoint << ";\n"; + out << SP << opName << "_params.zeroY = " << region.outputQuant.zeroPoint << ";\n"; + out << SP << opName << "_params.qminX = " << inputRange.first << ";\n"; + out << SP << opName << "_params.qmaxX = " << inputRange.second << ";\n"; + out << SP << opName << "_params.qminY = " << outputRange.first << ";\n"; + out << SP << opName << "_params.qmaxY = " << outputRange.second << ";\n"; + + const auto accumulatorScale = region.inputQuant.scale * region.weightQuant.scale; + const auto requantScale = accumulatorScale / region.outputQuant.scale; + std::int64_t requantMultiplier = 0; + int requantShift = 0; + const bool hasFixedPointRequantization = + INTERNAL::MakeQuantizedGemmFixedPointMultiplier(requantScale, requantMultiplier, requantShift); + + const bool hasBiasTensor = !context.biasTensor.empty(); + const bool hasAccumulatorBias = + !hasBiasTensor || + (region.biasQuant.has_value() && INTERNAL::NearlyEqualQuantizedScale(region.biasQuant->scale, accumulatorScale)); + std::int64_t biasRequantMultiplier = 0; + int biasRequantShift = 0; + const bool hasExactIntegerBias = + !hasBiasTensor || + (region.biasQuant.has_value() && INTERNAL::MakeExactIntegerScaleMultiplier( + region.biasQuant->scale / region.outputQuant.scale, biasRequantMultiplier, + biasRequantShift)); + const bool useIntegerEpilogue = hasFixedPointRequantization && (!hasBiasTensor || hasAccumulatorBias || hasExactIntegerBias); + if (useIntegerEpilogue) { + out << SP << opName << "_params.useIntegerEpilogue = true;\n"; + out << SP << opName << "_params.requantMultiplier = static_cast(" << requantMultiplier << ");\n"; + out << SP << opName << "_params.requantShift = " << requantShift << ";\n"; + if (hasBiasTensor && hasAccumulatorBias) { + out << SP << opName << "_params.useAccumulatorBias = true;\n"; + } else if (hasBiasTensor && hasExactIntegerBias) { + out << SP << opName << "_params.useIntegerBias = true;\n"; + out << SP << opName << "_params.biasRequantMultiplier = static_cast(" << biasRequantMultiplier << ");\n"; + out << SP << opName << "_params.biasRequantShift = " << biasRequantShift << ";\n"; + } + out << SP << "// Integer epilogue: fixed-point requantization"; + if (hasBiasTensor && hasAccumulatorBias) { + out << " with accumulator-domain bias"; + } else if (hasBiasTensor && hasExactIntegerBias) { + out << " with exact integer bias contribution"; + } + out << ".\n"; + } else { + out << SP << "// Integer epilogue disabled: QuantizedGemm_Call uses the exact float fallback.\n"; + } + + if (context.activation == EActivationType::RELU) { + out << SP << opName << "_params.activation = SOFIE::EQuantizedGemmActivation::Relu;\n"; + } else { + out << SP << opName << "_params.activation = SOFIE::EQuantizedGemmActivation::None;\n"; + } + + if (!context.biasTensor.empty()) { + const auto biasRange = QuantizedIntegerRange(*region.biasQuant); + out << SP << opName << "_params.hasBias = true;\n"; + out << SP << opName << "_params.scaleB = " + << std::setprecision(std::numeric_limits::max_digits10) << region.biasQuant->scale << ";\n"; + out << SP << opName << "_params.zeroB = " << region.biasQuant->zeroPoint << ";\n"; + out << SP << opName << "_params.qminB = " << biasRange.first << ";\n"; + out << SP << opName << "_params.qmaxB = " << biasRange.second << ";\n"; + } else { + out << SP << opName << "_params.hasBias = false;\n"; + } + + out << SP << "SOFIE::QuantizedGemm_Call(tensor_" << region.outputTensor << ", tensor_" + << region.inputSourceTensor << ", tensor_" << plan.weightStorageTensor << ", "; + if (!context.biasTensor.empty()) { + out << "tensor_" << region.biasSourceTensor; + } else { + out << "nullptr"; + } + out << ", " << opName << "_params);\n"; + + return out.str(); +} + + +inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantKernel(std::string opName, + const QuantizedGemmCodegenContext &context) +{ + INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm Alpaka fake-quant kernel"); + + std::stringstream out; + out << "\n//--------- ROperator_QuantizedGemm Alpaka fake-quant kernel " << opName << "\n"; + out << " struct QuantizedGemmAlpakaFakeQuantKernel_" << opName << " {\n"; + out << " template\n"; + out << " ALPAKA_FN_ACC void operator()(TAcc const & acc, const float * input, const float * weight, const float * bias, float * output,\n"; + out << " std::size_t elements, std::size_t n, std::size_t k, bool hasBias, bool hasRelu,\n"; + out << " double scaleX, double scaleW, double scaleB, double scaleY,\n"; + out << " std::int32_t zeroX, std::int32_t zeroW, std::int32_t zeroB, std::int32_t zeroY,\n"; + out << " std::int32_t qminX, std::int32_t qmaxX, std::int32_t qminW, std::int32_t qmaxW,\n"; + out << " std::int32_t qminB, std::int32_t qmaxB, std::int32_t qminY, std::int32_t qmaxY) const {\n"; + out << " const auto idx = alpaka::getIdx(acc)[0];\n"; + out << " if (idx >= elements) return;\n"; + out << " const std::size_t row = static_cast(idx) / n;\n"; + out << " const std::size_t col = static_cast(idx) % n;\n"; + out << " std::int32_t dot = 0;\n"; + out << " for (std::size_t kk = 0; kk < k; ++kk) {\n"; + out << " auto xReal = static_cast(input[row * k + kk]) / scaleX + static_cast(zeroX);\n"; + out << " auto wReal = static_cast(weight[col * k + kk]) / scaleW + static_cast(zeroW);\n"; + out << " auto xq = static_cast((xReal >= 0.0) ? (xReal + 0.5) : (xReal - 0.5));\n"; + out << " auto wq = static_cast((wReal >= 0.0) ? (wReal + 0.5) : (wReal - 0.5));\n"; + out << " xq = (xq < qminX) ? qminX : ((xq > qmaxX) ? qmaxX : xq);\n"; + out << " wq = (wq < qminW) ? qminW : ((wq > qmaxW) ? qmaxW : wq);\n"; + out << " dot += (xq - zeroX) * (wq - zeroW);\n"; + out << " }\n"; + out << " double real = static_cast(dot) * scaleX * scaleW;\n"; + out << " if (hasBias) {\n"; + out << " auto bReal = static_cast(bias[col]) / scaleB + static_cast(zeroB);\n"; + out << " auto bq = static_cast((bReal >= 0.0) ? (bReal + 0.5) : (bReal - 0.5));\n"; + out << " bq = (bq < qminB) ? qminB : ((bq > qmaxB) ? qmaxB : bq);\n"; + out << " real += static_cast(bq - zeroB) * scaleB;\n"; + out << " }\n"; + out << " auto yReal = real / scaleY + static_cast(zeroY);\n"; + out << " auto yq = static_cast((yReal >= 0.0) ? (yReal + 0.5) : (yReal - 0.5));\n"; + out << " if (hasRelu && yq < zeroY) yq = zeroY;\n"; + out << " yq = (yq < qminY) ? qminY : ((yq > qmaxY) ? qmaxY : yq);\n"; + out << " output[row * n + col] = static_cast(static_cast(yq - zeroY) * scaleY);\n"; + out << " }\n"; + out << " };\n"; + return out.str(); +} + +inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantDefinition(std::string opName, + const QuantizedGemmCodegenContext &context) +{ + INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm Alpaka fake-quant definition"); + return " QuantizedGemmAlpakaFakeQuantKernel_" + opName + " quantizedGemmAlpakaFakeQuantKernel_" + opName + ";\n"; +} + +inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string opName, + const QuantizedGemmCodegenContext &context, + const QuantizedGemmRegion ®ion, + const QuantizedLoweringPlan &plan) +{ + INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm Alpaka fake-quant launch"); + if (plan.backend != EQuantizedBackend::ALPAKA || !IsQuantizedLoweringAvailable(plan.status)) { + throw std::runtime_error("SOFIE fused Quantized Gemm Alpaka fake-quant launch requires an available Alpaka plan"); + } + if (region.inputSourceTensor.empty() || region.weightSourceTensor.empty() || region.outputTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm Alpaka fake-quant launch is missing quantization source/output tensors"); + } + if (!context.biasTensor.empty() && (region.biasSourceTensor.empty() || !region.biasQuant.has_value())) { + throw std::runtime_error("SOFIE fused Quantized Gemm Alpaka fake-quant launch is missing quantized bias metadata"); + } + + const auto dimA = context.inputShape.size(); + const auto dimB = context.weightShape.size(); + const auto m = context.inputShape[dimA - 2].GetVal(); + const auto k = context.inputShape[dimA - 1].GetVal(); + const auto n = context.weightShape[dimB - 2].GetVal(); + const auto elements = m + " * " + n; + const auto inputRange = QuantizedIntegerRange(region.inputQuant); + const auto weightRange = QuantizedIntegerRange(region.weightQuant); + const auto outputRange = QuantizedIntegerRange(region.outputQuant); + const auto biasRange = region.biasQuant ? QuantizedIntegerRange(*region.biasQuant) : std::pair{0, 0}; + + std::stringstream out; + out << "\n//--------- ROperator_QuantizedGemm Alpaka fake-quant launch " << opName << "\n"; + out << " // Fake-quant GPU path: one Alpaka thread computes one quantized GEMM output element.\n"; + out << " auto const elementsPerThread_quantizedGemm_" << opName << " = Vec::all(static_cast(1));\n"; + out << " auto const elementsPerGrid_quantizedGemm_" << opName << " = Vec::all(Idx{" << elements << "});\n"; + out << " auto const workDiv_quantizedGemm_" << opName << " = sofie_workdiv(elementsPerGrid_quantizedGemm_" << opName << ");\n"; + out << " auto task_op_" << opName << " = alpaka::createTaskKernel(workDiv_quantizedGemm_" << opName + << ", quantizedGemmAlpakaFakeQuantKernel_" << opName + << ", alpaka::getPtrNative(deviceBuf_" << region.inputSourceTensor << ")" + << ", alpaka::getPtrNative(deviceBuf_" << region.weightSourceTensor << ")"; + if (!context.biasTensor.empty()) { + out << ", alpaka::getPtrNative(deviceBuf_" << region.biasSourceTensor << ")"; + } else { + out << ", static_cast(nullptr)"; + } + out << ", alpaka::getPtrNative(deviceBuf_" << region.outputTensor << ")" + << ", static_cast(" << elements << ")" + << ", static_cast(" << n << ")" + << ", static_cast(" << k << ")" + << ", " << (!context.biasTensor.empty() ? "true" : "false") + << ", " << (context.activation == EActivationType::RELU ? "true" : "false") + << ", " << std::setprecision(std::numeric_limits::max_digits10) << region.inputQuant.scale + << ", " << std::setprecision(std::numeric_limits::max_digits10) << region.weightQuant.scale + << ", " << std::setprecision(std::numeric_limits::max_digits10) << (region.biasQuant ? region.biasQuant->scale : 1.0) + << ", " << std::setprecision(std::numeric_limits::max_digits10) << region.outputQuant.scale + << ", static_cast(" << region.inputQuant.zeroPoint << ")" + << ", static_cast(" << region.weightQuant.zeroPoint << ")" + << ", static_cast(" << (region.biasQuant ? region.biasQuant->zeroPoint : 0) << ")" + << ", static_cast(" << region.outputQuant.zeroPoint << ")" + << ", static_cast(" << inputRange.first << ")" + << ", static_cast(" << inputRange.second << ")" + << ", static_cast(" << weightRange.first << ")" + << ", static_cast(" << weightRange.second << ")" + << ", static_cast(" << biasRange.first << ")" + << ", static_cast(" << biasRange.second << ")" + << ", static_cast(" << outputRange.first << ")" + << ", static_cast(" << outputRange.second << "));\n"; + out << " alpaka::enqueue(queue, task_op_" << opName << ");\n"; + return out.str(); +} + +class ROperator_QuantizedGemm final : public ROperator { +private: + QuantizedGemmRegion fRegion; + QuantizedLoweringPlan fPlan; + QuantizedGemmCodegenContext fContext; + +public: + ROperator_QuantizedGemm(QuantizedGemmRegion region, QuantizedLoweringPlan plan, + QuantizedGemmCodegenContext context) + : fRegion(std::move(region)), fPlan(std::move(plan)), fContext(std::move(context)) + { + fKind = OperatorKind::QUANTIZED_GEMM; + fName = "QuantizedGemm"; + fInputTensorNames = { fRegion.inputSourceTensor, fRegion.weightSourceTensor }; + if (!fRegion.biasSourceTensor.empty()) { + fInputTensorNames.emplace_back(fRegion.biasSourceTensor); + } + fOutputTensorNames = { fRegion.outputTensor }; + } + + std::vector GetStdLibs() override { return { "cmath", "cstdint", "vector" }; } + + void Initialize(RModel &) override {} + + std::string Generate(std::string opName) override + { + if (fPlan.backend != EQuantizedBackend::CPU || !IsQuantizedLoweringAvailable(fPlan.status) || + !fPlan.suppressesGraphOperators || !fPlan.usesPrequantizedWeights || + fPlan.weightLayout != EQuantizedLayout::PackedCPU) { + throw std::runtime_error("SOFIE ROperator_QuantizedGemm CPU code generation requires an available packed-weight CPU lowering plan"); + } + std::string code = "\n//--------- ROperator_QuantizedGemm synthetic fused CPU operator " + opName + "\n"; + return code + GenerateFusedQuantizedGemmCallCPUCode(std::move(opName), fContext, fRegion, fPlan); + } + + std::string Generate_GPU_Kernel_ALPAKA(std::string opName) override + { + return GenerateFusedQuantizedGemmAlpakaFakeQuantKernel(std::move(opName), fContext); + } + + std::string Generate_GPU_Kernel_Definitions_ALPAKA(std::string opName) override + { + return GenerateFusedQuantizedGemmAlpakaFakeQuantDefinition(std::move(opName), fContext); + } + + std::string Generate_GPU_ALPAKA(std::string opName) override + { + return GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::move(opName), fContext, fRegion, fPlan); + } +}; + +} // namespace SOFIE + +#endif // SOFIE_ROPERATOR_QUANTIZED_GEMM diff --git a/core/inc/SOFIE/RQuantization.hxx b/core/inc/SOFIE/RQuantization.hxx new file mode 100644 index 0000000..3d8667a --- /dev/null +++ b/core/inc/SOFIE/RQuantization.hxx @@ -0,0 +1,225 @@ +#ifndef SOFIE_RQUANTIZATION +#define SOFIE_RQUANTIZATION + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +enum class EQuantizationRoundingMode { + UNDEFINED = 0, ROUND = 1, FLOOR = 2, TRUNCATE = 3 +}; + +enum class EQuantizationOverflowMode { + UNDEFINED = 0, SAT = 1, SAT_SYM = 2 +}; + +enum class EQuantizationGranularity { + UNDEFINED = 0, PerTensor = 1, PerChannel = 2, PerWeight = 3 +}; + +struct QuantizationInfo { + unsigned bitWidth = 0; + bool isSigned = false; + bool narrow = false; + double scale = 1.0; + std::int64_t zeroPoint = 0; + EQuantizationRoundingMode rounding = EQuantizationRoundingMode::UNDEFINED; + EQuantizationOverflowMode overflow = EQuantizationOverflowMode::UNDEFINED; + EQuantizationGranularity granularity = EQuantizationGranularity::PerTensor; + int axis = -1; +}; + +inline std::pair QuantizedIntegerRange(const QuantizationInfo &info) +{ + if (info.bitWidth == 0 || info.bitWidth >= 63) { + throw std::runtime_error("SOFIE quantization metadata received unsupported bit width"); + } + if (info.isSigned) { + const std::int64_t qmax = (std::int64_t{1} << (info.bitWidth - 1)) - 1; + const std::int64_t qmin = info.narrow ? -qmax : -(std::int64_t{1} << (info.bitWidth - 1)); + return {qmin, qmax}; + } + const std::int64_t qmax = (std::int64_t{1} << info.bitWidth) - 1; + const std::int64_t qmin = info.narrow ? 1 : 0; + return {qmin, qmax}; +} + +inline std::int64_t QuantizeScalarToIntegerGrid(float value, const QuantizationInfo &info) +{ + const auto range = QuantizedIntegerRange(info); + auto quantized = static_cast(std::llround((static_cast(value) / info.scale) + info.zeroPoint)); + if (quantized < range.first) + quantized = range.first; + if (quantized > range.second) + quantized = range.second; + return quantized; +} + +enum class EQuantizedLoweringStatus { + UNDEFINED = 0, + Optimized = 1, + Baseline = 2, + BackendUnsupported = 3, + SemanticUnsupported = 4, + SemanticRecognized = 5 +}; + +inline bool IsQuantizedLoweringAvailable(EQuantizedLoweringStatus status) +{ + return status == EQuantizedLoweringStatus::Optimized || status == EQuantizedLoweringStatus::Baseline; +} + +inline bool IsQuantizedLoweringUnsupported(EQuantizedLoweringStatus status) +{ + return status == EQuantizedLoweringStatus::BackendUnsupported || + status == EQuantizedLoweringStatus::SemanticUnsupported; +} + +enum class EQuantizedBackend { + UNDEFINED = 0, CPU = 1, ALPAKA = 2 +}; + +enum class EQuantizedStorageType { + UNDEFINED = 0, FloatCarrier = 1, Int8 = 2, UInt8 = 3, Int32Accumulator = 4, MetadataOnly = 5 +}; + +enum class EQuantizedLayout { + UNDEFINED = 0, Plain = 1, Transposed = 2, PackedCPU = 3, TiledAlpaka = 4 +}; + +struct QuantizedTensorStorage { + std::string logicalTensor; + std::string sourceTensor; + std::string storageTensor; + EQuantizedStorageType storageType = EQuantizedStorageType::UNDEFINED; + EQuantizedLayout layout = EQuantizedLayout::UNDEFINED; + QuantizationInfo quantization; + std::vector shape; + + bool isConstant = false; + bool isPersistent = true; + bool isTransient = false; + bool isDeviceResident = false; + std::size_t byteSize = 0; +}; + +inline std::size_t QuantizedStorageElementSize(EQuantizedStorageType type) +{ + switch (type) { + case EQuantizedStorageType::FloatCarrier: + return sizeof(float); + case EQuantizedStorageType::Int8: + return sizeof(std::int8_t); + case EQuantizedStorageType::UInt8: + return sizeof(std::uint8_t); + case EQuantizedStorageType::Int32Accumulator: + return sizeof(std::int32_t); + default: + return 0; + } +} + +inline std::size_t QuantizedStorageElementCount(const std::vector &shape) +{ + if (shape.empty()) + return 0; + std::size_t count = 1; + for (auto dim : shape) + count *= dim; + return count; +} + +inline std::size_t QuantizedStorageByteSize(EQuantizedStorageType type, const std::vector &shape) +{ + return QuantizedStorageElementSize(type) * QuantizedStorageElementCount(shape); +} + +inline bool IsPhysicalQuantizedStorage(EQuantizedStorageType type) +{ + return type == EQuantizedStorageType::Int8 || type == EQuantizedStorageType::UInt8 || + type == EQuantizedStorageType::Int32Accumulator; +} + +struct QuantizedLoweringPlan { + EQuantizedBackend backend = EQuantizedBackend::UNDEFINED; + EQuantizedLoweringStatus status = EQuantizedLoweringStatus::UNDEFINED; + std::string reason; + + EQuantizedStorageType inputStorage = EQuantizedStorageType::UNDEFINED; + EQuantizedStorageType weightStorage = EQuantizedStorageType::UNDEFINED; + EQuantizedStorageType biasStorage = EQuantizedStorageType::UNDEFINED; + EQuantizedStorageType accumulatorStorage = EQuantizedStorageType::UNDEFINED; + EQuantizedStorageType outputStorage = EQuantizedStorageType::UNDEFINED; + + std::string weightStorageTensor; + EQuantizedLayout weightLayout = EQuantizedLayout::UNDEFINED; + + std::vector consumedOperatorIndices; + bool preservesQuantizationSemantics = false; + bool hasBaselineLowering = false; + bool hasOptimizedLowering = false; + bool isMetadataOnly = false; + bool usesInt32Accumulator = false; + bool usesPrequantizedWeights = false; + bool suppressesGraphOperators = false; +}; + +struct QuantizedGemmRegion { + // Quantized carrier tensors, i.e. outputs of quantization boundaries. + std::string inputTensor; + std::string weightTensor; + std::string biasTensor; + std::string gemmOutputTensor; + std::string outputTensor; + + // Source tensors consumed by the quantization boundaries. Fused region code + // uses these names when it suppresses the literal Quant nodes. + std::string inputSourceTensor; + std::string weightSourceTensor; + std::string biasSourceTensor; + + std::size_t inputQuantOpIndex = static_cast(-1); + std::size_t weightQuantOpIndex = static_cast(-1); + std::optional biasQuantOpIndex; + std::size_t gemmOpIndex = static_cast(-1); + std::size_t outputQuantOpIndex = static_cast(-1); + + QuantizationInfo inputQuant; + QuantizationInfo weightQuant; + std::optional biasQuant; + QuantizationInfo outputQuant; + + float alpha = 1.0f; + float beta = 1.0f; + std::int64_t transA = 0; + std::int64_t transB = 0; + + EQuantizedLoweringStatus status = EQuantizedLoweringStatus::UNDEFINED; + std::string reason; +}; + +struct QuantizationModelState { + std::unordered_map tensorInfos; + std::unordered_map tensorStorages; + std::unordered_map gemmRegions; + std::unordered_map> loweringPlans; + + void ClearDerivedAnalysis() + { + tensorStorages.clear(); + gemmRegions.clear(); + loweringPlans.clear(); + } +}; + +} // namespace SOFIE + +#endif // SOFIE_RQUANTIZATION diff --git a/core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx b/core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx new file mode 100644 index 0000000..2d6c3da --- /dev/null +++ b/core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx @@ -0,0 +1,268 @@ +#ifndef SOFIE_SOFIE_QUANTIZED_RUNTIME +#define SOFIE_SOFIE_QUANTIZED_RUNTIME + +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +// GEMM only, need to abstract for multiple operators / backends. + +enum class EQuantizedGemmActivation { + None = 0, + Relu = 1 +}; + +struct QuantizedGemmParams { + std::size_t m = 0; + std::size_t n = 0; + std::size_t k = 0; + std::size_t tileN = 4; + + double scaleX = 1.0; + double scaleW = 1.0; + double scaleY = 1.0; + double scaleB = 1.0; + + std::int32_t zeroX = 0; + std::int32_t zeroW = 0; + std::int32_t zeroY = 0; + std::int32_t zeroB = 0; + + std::int32_t qminX = 0; + std::int32_t qmaxX = 0; + std::int32_t qminY = 0; + std::int32_t qmaxY = 0; + std::int32_t qminB = 0; + std::int32_t qmaxB = 0; + + bool hasBias = false; + + bool useIntegerEpilogue = false; + bool useAccumulatorBias = false; + bool useIntegerBias = false; + std::int64_t requantMultiplier = 0; + int requantShift = 0; + std::int64_t biasRequantMultiplier = 0; + int biasRequantShift = 0; + + EQuantizedGemmActivation activation = EQuantizedGemmActivation::None; +}; + +namespace INTERNAL { + +enum class EQuantizedGemmCPUBackend { + Portable = 0, + X86 = 1, + ARM = 2 +}; + +inline constexpr bool IsQuantizedGemmX86Target() +{ +#if defined(__x86_64__) || defined(_M_X64) || defined(__i386) || defined(_M_IX86) + return true; +#else + return false; +#endif +} + +inline constexpr bool IsQuantizedGemmARMTarget() +{ +#if defined(__aarch64__) || defined(__arm__) || defined(_M_ARM64) || defined(_M_ARM) + return true; +#else + return false; +#endif +} + +inline constexpr EQuantizedGemmCPUBackend SelectQuantizedGemmCPUBackend() +{ + // Selects the CPU backend family visible at compile time. + // Portable remains the fallback for every target family. + if constexpr (IsQuantizedGemmX86Target()) { + return EQuantizedGemmCPUBackend::X86; + } else if constexpr (IsQuantizedGemmARMTarget()) { + return EQuantizedGemmCPUBackend::ARM; + } else { + return EQuantizedGemmCPUBackend::Portable; + } +} + +inline std::int32_t QuantizedGemmClamp(std::int32_t value, std::int32_t qmin, std::int32_t qmax) +{ + return std::max(qmin, std::min(qmax, value)); +} + +inline std::int32_t QuantizedGemmQuantizeClamp(double value, double scale, std::int32_t zero, std::int32_t qmin, + std::int32_t qmax) +{ + auto quantized = static_cast(std::nearbyint((value / scale) + zero)); + return QuantizedGemmClamp(quantized, qmin, qmax); +} + +inline std::int64_t QuantizedGemmRoundDivideByPowerOfTwo(std::int64_t value, int shift) +{ + if (shift <= 0) { + const auto leftShift = -shift; + if (leftShift >= 62) { + return value >= 0 ? std::numeric_limits::max() : std::numeric_limits::min(); + } + return value << leftShift; + } + + if (shift >= 62) { + return 0; + } + + const std::uint64_t magnitude = value < 0 ? static_cast(-value) : static_cast(value); + const std::uint64_t quotient = magnitude >> shift; + const std::uint64_t remainder = magnitude & ((std::uint64_t{1} << shift) - 1); + const std::uint64_t half = std::uint64_t{1} << (shift - 1); + const bool roundUp = (remainder > half) || (remainder == half && (quotient & std::uint64_t{1}) != 0); + const auto rounded = static_cast(quotient + (roundUp ? 1 : 0)); + return value < 0 ? -rounded : rounded; +} + +inline std::int64_t QuantizedGemmApplyFixedPointMultiplier(std::int32_t accumulator, std::int64_t multiplier, int shift) +{ + return QuantizedGemmRoundDivideByPowerOfTwo(static_cast(accumulator) * multiplier, shift); +} + +inline std::int32_t QuantizedGemmFinalizeY(std::int64_t yq, const QuantizedGemmParams ¶ms) +{ + if (params.activation == EQuantizedGemmActivation::Relu) { + yq = std::max(yq, params.zeroY); + } + yq = std::max(params.qminY, std::min(params.qmaxY, yq)); + return static_cast(yq); +} + +inline float QuantizedGemmFinalizeAccumulator(std::int32_t accumulator, std::size_t col, const float *bias, + const QuantizedGemmParams ¶ms) +{ + const bool hasRuntimeBias = params.hasBias && bias != nullptr; + const bool canUseIntegerEpilogue = + params.useIntegerEpilogue && (!hasRuntimeBias || params.useAccumulatorBias || params.useIntegerBias); + + if (canUseIntegerEpilogue) { + auto acc = accumulator; + std::int32_t biasCentered = 0; + if (hasRuntimeBias) { + const auto bq = QuantizedGemmQuantizeClamp(static_cast(bias[col]), params.scaleB, params.zeroB, + params.qminB, params.qmaxB); + biasCentered = bq - params.zeroB; + if (params.useAccumulatorBias) { + acc += biasCentered; + } + } + + auto scaled = QuantizedGemmApplyFixedPointMultiplier(acc, params.requantMultiplier, params.requantShift); + if (hasRuntimeBias && params.useIntegerBias && !params.useAccumulatorBias) { + scaled += QuantizedGemmApplyFixedPointMultiplier(biasCentered, params.biasRequantMultiplier, + params.biasRequantShift); + } + const auto yq = QuantizedGemmFinalizeY(scaled + params.zeroY, params); + return static_cast(static_cast(yq - params.zeroY) * params.scaleY); + } + + double real = static_cast(accumulator) * params.scaleX * params.scaleW; + if (hasRuntimeBias) { + const auto bq = QuantizedGemmQuantizeClamp(static_cast(bias[col]), params.scaleB, params.zeroB, + params.qminB, params.qmaxB); + real += static_cast(bq - params.zeroB) * params.scaleB; + } + + const auto yq = QuantizedGemmFinalizeY( + static_cast(std::nearbyint((real / params.scaleY) + params.zeroY)), params); + return static_cast(static_cast(yq - params.zeroY) * params.scaleY); +} + +template +inline void QuantizedGemmCPU_Portable(float *output, const float *input, const WeightT *packedWeight, const float *bias, + const QuantizedGemmParams ¶ms) +{ + if (params.tileN == 0) { + return; + } + + thread_local std::vector xqScratch; + xqScratch.resize(params.k); + auto *xqRow = xqScratch.data(); + + for (std::size_t row = 0; row < params.m; ++row) { + for (std::size_t kk = 0; kk < params.k; ++kk) { + xqRow[kk] = QuantizedGemmQuantizeClamp(static_cast(input[row * params.k + kk]), params.scaleX, + params.zeroX, params.qminX, params.qmaxX); + } + + for (std::size_t colBase = 0; colBase < params.n; colBase += params.tileN) { + const std::size_t block = colBase / params.tileN; + const std::size_t columns = std::min(params.tileN, params.n - colBase); + + if (params.tileN == 4) { + std::int32_t acc[4] = {0, 0, 0, 0}; + + for (std::size_t kk = 0; kk < params.k; ++kk) { + const auto xCentered = xqRow[kk] - params.zeroX; + const auto weightBase = (block * params.k + kk) * params.tileN; + for (std::size_t ji = 0; ji < columns; ++ji) { + const auto wq = static_cast(packedWeight[weightBase + ji]); + acc[ji] += xCentered * (wq - params.zeroW); + } + } + + for (std::size_t ji = 0; ji < columns; ++ji) { + const std::size_t col = colBase + ji; + output[row * params.n + col] = QuantizedGemmFinalizeAccumulator(acc[ji], col, bias, params); + } + } else { + std::vector acc(params.tileN); + + for (std::size_t kk = 0; kk < params.k; ++kk) { + const auto xCentered = xqRow[kk] - params.zeroX; + const auto weightBase = (block * params.k + kk) * params.tileN; + for (std::size_t ji = 0; ji < columns; ++ji) { + const auto wq = static_cast(packedWeight[weightBase + ji]); + acc[ji] += xCentered * (wq - params.zeroW); + } + } + + for (std::size_t ji = 0; ji < columns; ++ji) { + const std::size_t col = colBase + ji; + output[row * params.n + col] = QuantizedGemmFinalizeAccumulator(acc[ji], col, bias, params); + } + } + } + } +} + +template +inline void QuantizedGemmCPU_Dispatch(float *output, const float *input, const WeightT *packedWeight, const float *bias, + const QuantizedGemmParams ¶ms) +{ + [[maybe_unused]] constexpr auto backend = SelectQuantizedGemmCPUBackend(); + // The dispatch boundary is explicit even when the portable kernel is selected. + // Architecture-specific kernels can replace this call without changing generated code. + QuantizedGemmCPU_Portable(output, input, packedWeight, bias, params); +} + +} // namespace INTERNAL + +template +inline void QuantizedGemm_Call(float *output, const float *input, const WeightT *packedWeight, const float *bias, + const QuantizedGemmParams ¶ms) +{ + static_assert(std::is_same_v || std::is_same_v, + "SOFIE::QuantizedGemm_Call supports int8/uint8 packed weights"); + + INTERNAL::QuantizedGemmCPU_Dispatch(output, input, packedWeight, bias, params); +} + +} // namespace SOFIE + +#endif // SOFIE_SOFIE_QUANTIZED_RUNTIME diff --git a/core/inc/SOFIE/SOFIE_common.hxx b/core/inc/SOFIE/SOFIE_common.hxx index e36df0a..a2d533f 100644 --- a/core/inc/SOFIE/SOFIE_common.hxx +++ b/core/inc/SOFIE/SOFIE_common.hxx @@ -20,6 +20,7 @@ #include #include #include +#include namespace SOFIE { diff --git a/core/src/RModel.cxx b/core/src/RModel.cxx index 377171c..4b4a3f0 100644 --- a/core/src/RModel.cxx +++ b/core/src/RModel.cxx @@ -3,6 +3,8 @@ #include #include #include +#include +#include #ifdef SOFIE_SUPPORT_ROOT_BINARY #include "TFile.h" @@ -52,6 +54,13 @@ std::string TensorMember(std::string const &name) } // namespace +void RModel::BuildLoweredOperatorView(EQuantizedBackend backend) +{ + fLoweredOperators.clear(); + fLoweredConsumedOperatorIndices.clear(); + AddLoweredQuantizedGemmOperators(backend); +} + std::vector RModel::GetTensorShape(const std::string & name) const { auto f = fReadyInputTensorInfos.find(name); if (f != fReadyInputTensorInfos.end()) { @@ -677,6 +686,8 @@ void RModel::Initialize(const std::map & inputParams, bool i++; } + AnalyzeQuantizedRegions(); + // loop on initialized tensors and make the integers as constant to be // not written in a weight file and check if the tensors flagged as not writable are really not writable, // i.e. are not used by non constant operators @@ -823,7 +834,10 @@ void RModel::GenerateInitializedTensorInfo() } else if (i.second.type() == ETensorType::INT32) { fGC += GenerateConstantTensorCode(i); fConstantTensorSize += length * sizeof(int32_t); - } else if (i.second.type() == ETensorType::BOOL || i.second.type() == ETensorType::UINT8 ) { + } else if (i.second.type() == ETensorType::INT8) { + fGC += GenerateConstantTensorCode(i); + fConstantTensorSize += length * sizeof(int8_t); + } else if (i.second.type() == ETensorType::BOOL || i.second.type() == ETensorType::UINT8 ) { fGC += GenerateConstantTensorCode(i); fConstantTensorSize += length * sizeof(uint8_t); } @@ -1439,8 +1453,13 @@ void RModel::GenerateSessionCode() for (size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx) { if (fVerbose) std::cout << "Generating code for operator .... " << op_idx << std::endl; + if (!fProfile && fLoweredConsumedOperatorIndices.count(op_idx) != 0) { + continue; + } if (fProfile) { allOperatorCode += RModelProfiler::GenerateOperatorCode(*fOperators[op_idx], op_idx); + } else if (auto lowered = fLoweredOperators.find(op_idx); lowered != fLoweredOperators.end()) { + allOperatorCode += lowered->second->Generate(std::to_string(op_idx)); } else { allOperatorCode += fOperators[op_idx]->Generate(std::to_string(op_idx)); } @@ -1519,6 +1538,9 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, lo // initialize the model including all operators and sub-graphs Initialize(batchSize, verbose); + BuildLoweredOperatorView(EQuantizedBackend::CPU); + + AddQuantizedGeneratedHeaders(); // if having dynamic tensor we need to have a Session if (!fDynamicTensorInfos.empty()) { diff --git a/core/src/RModel_Quantization.cxx b/core/src/RModel_Quantization.cxx new file mode 100644 index 0000000..f54dda4 --- /dev/null +++ b/core/src/RModel_Quantization.cxx @@ -0,0 +1,556 @@ +#include "SOFIE/RModel.hxx" +#include "SOFIE/ROperator_Gemm.hxx" +#include "SOFIE/ROperator_QuantizedGemm.hxx" +#include "SOFIE/RQuantization.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +namespace { + +std::string JoinReasons(const std::vector &reasons) +{ + std::ostringstream out; + for (std::size_t i = 0; i < reasons.size(); ++i) { + if (i != 0) + out << "; "; + out << reasons[i]; + } + return out.str(); +} + +bool IsScalarPerTensor(const QuantizationInfo &info) +{ + return info.granularity == EQuantizationGranularity::PerTensor && info.axis == -1; +} + +EQuantizedStorageType StorageTypeForQuantizedTensor(const QuantizationInfo &info) +{ + return info.isSigned ? EQuantizedStorageType::Int8 : EQuantizedStorageType::UInt8; +} + +std::string GetQuantBoundarySourceTensor(const ROperator &op) +{ + auto inputs = op.GetOpInputTensors(); + if (inputs.empty()) { + return {}; + } + return std::string(inputs[0]); +} + +QuantizedGemmCodegenContext MakeQuantizedGemmCodegenContext(const ROperator_Gemm &gemm) +{ + QuantizedGemmCodegenContext context; + context.inputTensor = gemm.GetInputTensorName(); + context.weightTensor = gemm.GetWeightTensorName(); + context.biasTensor = gemm.GetBiasTensorName(); + context.outputTensor = gemm.GetOutputTensorName(); + context.inputShape = gemm.GetInputShape(); + context.weightShape = gemm.GetWeightShape(); + context.outputShape = gemm.GetOutputShape(); + context.alpha = gemm.GetAlpha(); + context.beta = gemm.GetBeta(); + context.transA = gemm.GetTransA(); + context.transB = gemm.GetTransB(); + context.activation = gemm.GetActivationType(); + context.indent = " "; + return context; +} + +QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion ®ion, + const std::string &weightStorageTensor) +{ + QuantizedLoweringPlan plan; + plan.backend = EQuantizedBackend::CPU; + plan.status = EQuantizedLoweringStatus::Baseline; + plan.reason = "CPU baseline lowering with packed pre-quantized weight storage"; + plan.inputStorage = StorageTypeForQuantizedTensor(region.inputQuant); + plan.weightStorage = StorageTypeForQuantizedTensor(region.weightQuant); + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.weightStorageTensor = weightStorageTensor; + plan.weightLayout = EQuantizedLayout::PackedCPU; + plan.consumedOperatorIndices = { region.inputQuantOpIndex, region.weightQuantOpIndex, region.gemmOpIndex, region.outputQuantOpIndex }; + if (region.biasQuantOpIndex) { + plan.consumedOperatorIndices.push_back(*region.biasQuantOpIndex); + } + std::sort(plan.consumedOperatorIndices.begin(), plan.consumedOperatorIndices.end()); + plan.preservesQuantizationSemantics = true; + plan.hasBaselineLowering = true; + plan.hasOptimizedLowering = false; + plan.isMetadataOnly = false; + plan.usesInt32Accumulator = true; + plan.usesPrequantizedWeights = true; + plan.suppressesGraphOperators = true; + return plan; +} + +QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend, std::string reason, bool preservesSemantics) +{ + QuantizedLoweringPlan plan; + plan.backend = backend; + plan.status = preservesSemantics ? EQuantizedLoweringStatus::BackendUnsupported + : EQuantizedLoweringStatus::SemanticUnsupported; + plan.reason = std::move(reason); + plan.inputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.weightStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.biasStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.accumulatorStorage = EQuantizedStorageType::UNDEFINED; + plan.outputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.preservesQuantizationSemantics = preservesSemantics; + plan.hasBaselineLowering = false; + plan.hasOptimizedLowering = false; + plan.isMetadataOnly = preservesSemantics; + plan.usesInt32Accumulator = false; + plan.usesPrequantizedWeights = false; + plan.suppressesGraphOperators = false; + return plan; +} + +QuantizedLoweringPlan MakeAlpakaFakeQuantPlan(const QuantizedGemmRegion ®ion) +{ + QuantizedLoweringPlan plan; + plan.backend = EQuantizedBackend::ALPAKA; + plan.status = EQuantizedLoweringStatus::Baseline; + plan.reason = "Alpaka fake-quant lowering over float carrier tensors"; + plan.inputStorage = EQuantizedStorageType::FloatCarrier; + plan.weightStorage = EQuantizedStorageType::FloatCarrier; + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.weightLayout = EQuantizedLayout::Plain; + plan.consumedOperatorIndices = { region.inputQuantOpIndex, region.weightQuantOpIndex, region.gemmOpIndex, region.outputQuantOpIndex }; + if (region.biasQuantOpIndex) { + plan.consumedOperatorIndices.push_back(*region.biasQuantOpIndex); + } + std::sort(plan.consumedOperatorIndices.begin(), plan.consumedOperatorIndices.end()); + plan.preservesQuantizationSemantics = true; + plan.hasBaselineLowering = true; + plan.hasOptimizedLowering = false; + plan.isMetadataOnly = false; + plan.usesInt32Accumulator = true; + plan.usesPrequantizedWeights = false; + plan.suppressesGraphOperators = true; + return plan; +} + +void CheckQuantInfo(const QuantizationInfo &info, const std::string &role, bool requireSigned, + std::vector &reasons) +{ + if (!IsScalarPerTensor(info)) { + reasons.push_back(role + " quantization is not scalar per-tensor"); + } + if (info.bitWidth == 0 || info.bitWidth > 8) { + reasons.push_back(role + " bit width is not in the initially supported range [1, 8]"); + } + if (requireSigned && !info.isSigned) { + reasons.push_back(role + " quantization is not signed"); + } + if (info.zeroPoint != 0) { + reasons.push_back(role + " zero point is not 0"); + } + if (info.scale <= 0.0 || !std::isfinite(info.scale)) { + reasons.push_back(role + " scale is not positive and finite"); + } + if (info.rounding != EQuantizationRoundingMode::ROUND) { + reasons.push_back(role + " rounding mode is not ROUND"); + } + if (info.overflow != EQuantizationOverflowMode::SAT && info.overflow != EQuantizationOverflowMode::SAT_SYM) { + reasons.push_back(role + " overflow mode is unsupported"); + } +} + +} // namespace + +void RModel::AddQuantizationInfo(const std::string & tensor_name, QuantizationInfo info) +{ + fQuantizationState.tensorInfos[UTILITY::Clean_name(tensor_name)] = std::move(info); +} + +bool RModel::HasQuantizationInfo(const std::string & tensor_name) const +{ + auto clean_name = UTILITY::Clean_name(tensor_name); + if (fQuantizationState.tensorInfos.find(clean_name) != fQuantizationState.tensorInfos.end()) + return true; + if (fIsSubGraph && fParentGraph) + return fParentGraph->HasQuantizationInfo(clean_name); + return false; +} + +const QuantizationInfo & RModel::GetQuantizationInfo(const std::string & tensor_name) const +{ + auto clean_name = UTILITY::Clean_name(tensor_name); + auto f = fQuantizationState.tensorInfos.find(clean_name); + if (f != fQuantizationState.tensorInfos.end()) + return f->second; + if (fIsSubGraph && fParentGraph) + return fParentGraph->GetQuantizationInfo(clean_name); + throw std::runtime_error("SOFIE tensor [" + clean_name + "] has no quantization information"); +} + + +bool RModel::HasQuantizedGemmRegion(std::size_t op_index) const +{ + return fQuantizationState.gemmRegions.find(op_index) != fQuantizationState.gemmRegions.end(); +} + +const QuantizedGemmRegion & RModel::GetQuantizedGemmRegion(std::size_t op_index) const +{ + auto it = fQuantizationState.gemmRegions.find(op_index); + if (it == fQuantizationState.gemmRegions.end()) { + throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no quantized Gemm information"); + } + return it->second; +} + +bool RModel::HasQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const +{ + auto opIt = fQuantizationState.loweringPlans.find(op_index); + return opIt != fQuantizationState.loweringPlans.end() && opIt->second.find(backend) != opIt->second.end(); +} + +const QuantizedLoweringPlan & RModel::GetQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const +{ + auto opIt = fQuantizationState.loweringPlans.find(op_index); + if (opIt == fQuantizationState.loweringPlans.end()) { + throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no quantized lowering plans"); + } + auto backendIt = opIt->second.find(backend); + if (backendIt == opIt->second.end()) { + throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no requested backend quantized lowering plan"); + } + return backendIt->second; +} + +void RModel::RegisterQuantizedTensorStorage(QuantizedTensorStorage storage) +{ + storage.storageTensor = UTILITY::Clean_name(storage.storageTensor); + storage.logicalTensor = UTILITY::Clean_name(storage.logicalTensor); + storage.sourceTensor = UTILITY::Clean_name(storage.sourceTensor); + if (storage.storageTensor.empty()) { + throw std::runtime_error("SOFIE quantized tensor storage registration requires a storage tensor name"); + } + fQuantizationState.tensorStorages[storage.storageTensor] = std::move(storage); +} + +bool RModel::HasQuantizedTensorStorage(const std::string & storage_tensor_name) const +{ + auto clean_name = UTILITY::Clean_name(storage_tensor_name); + return fQuantizationState.tensorStorages.find(clean_name) != fQuantizationState.tensorStorages.end(); +} + +const QuantizedTensorStorage & RModel::GetQuantizedTensorStorage(const std::string & storage_tensor_name) const +{ + auto clean_name = UTILITY::Clean_name(storage_tensor_name); + auto it = fQuantizationState.tensorStorages.find(clean_name); + if (it == fQuantizationState.tensorStorages.end()) { + throw std::runtime_error("SOFIE tensor [" + clean_name + "] has no quantized storage information"); + } + return it->second; +} + +std::vector RModel::GetQuantizedGemmOperatorIndices() const +{ + std::vector indices; + indices.reserve(fQuantizationState.gemmRegions.size()); + for (const auto &entry : fQuantizationState.gemmRegions) { + indices.push_back(entry.first); + } + std::sort(indices.begin(), indices.end()); + return indices; +} + +void RModel::AnalyzeQuantizedRegions() +{ + fQuantizationState.ClearDerivedAnalysis(); + + std::unordered_map producerByTensor; + std::unordered_map> consumersByTensor; + + for (std::size_t opIndex = 0; opIndex < fOperators.size(); ++opIndex) { + for (const auto &output : fOperators[opIndex]->GetOpOutputTensors()) { + producerByTensor[std::string(output)] = opIndex; + } + for (const auto &input : fOperators[opIndex]->GetOpInputTensors()) { + consumersByTensor[std::string(input)].push_back(opIndex); + } + } + + for (std::size_t opIndex = 0; opIndex < fOperators.size(); ++opIndex) { + if (fOperators[opIndex]->GetKind() != OperatorKind::GEMM) + continue; + + auto *gemm = dynamic_cast *>(fOperators[opIndex].get()); + if (!gemm) + continue; + + QuantizedGemmRegion info; + info.status = EQuantizedLoweringStatus::SemanticUnsupported; + info.alpha = gemm->GetAlpha(); + info.beta = gemm->GetBeta(); + info.transA = gemm->GetTransA(); + info.transB = gemm->GetTransB(); + info.gemmOpIndex = opIndex; + + std::vector reasons; + + auto inputs = gemm->GetOpInputTensors(); + auto outputs = gemm->GetOpOutputTensors(); + if (inputs.size() < 2 || outputs.size() != 1) { + reasons.push_back("Gemm does not have the expected input/output arity"); + } else { + info.inputTensor = std::string(inputs[0]); + info.weightTensor = std::string(inputs[1]); + if (inputs.size() >= 3) + info.biasTensor = std::string(inputs[2]); + info.gemmOutputTensor = std::string(outputs[0]); + } + + if (std::fabs(info.alpha - 1.0f) > 0.0f) + reasons.push_back("Gemm alpha is not 1"); + if (std::fabs(info.beta - 1.0f) > 0.0f) + reasons.push_back("Gemm beta is not 1"); + if (info.transA != 0) + reasons.push_back("Gemm transA is not 0"); + if (info.transB != 1) + reasons.push_back("Gemm transB is not 1"); + if (!info.inputTensor.empty() && GetTensorShape(info.inputTensor).size() != 2) + reasons.push_back("input tensor is not rank-2 for quantized Gemm lowering"); + if (!info.weightTensor.empty() && GetTensorShape(info.weightTensor).size() != 2) + reasons.push_back("weight tensor is not rank-2 for quantized Gemm lowering"); + if (!info.gemmOutputTensor.empty() && GetTensorShape(info.gemmOutputTensor).size() != 2) + reasons.push_back("Gemm output tensor is not rank-2 for quantized Gemm lowering"); + + const auto requireQuantProducer = [&](const std::string &tensor, const std::string &role) -> std::optional { + auto producer = producerByTensor.find(tensor); + if (producer == producerByTensor.end()) { + reasons.push_back(role + " tensor has no producer quantization boundary"); + return std::nullopt; + } + if (!fOperators[producer->second]->IsQuantizationBoundary()) { + reasons.push_back(role + " tensor producer is not a quantization boundary"); + return std::nullopt; + } + return producer->second; + }; + + if (!info.inputTensor.empty()) { + if (auto producer = requireQuantProducer(info.inputTensor, "input")) { + info.inputQuantOpIndex = *producer; + info.inputSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); + } + if (HasQuantizationInfo(info.inputTensor)) { + info.inputQuant = GetQuantizationInfo(info.inputTensor); + CheckQuantInfo(info.inputQuant, "input", false, reasons); + } else { + reasons.push_back("input tensor has no QuantizationInfo"); + } + } + + if (!info.weightTensor.empty()) { + if (auto producer = requireQuantProducer(info.weightTensor, "weight")) { + info.weightQuantOpIndex = *producer; + info.weightSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); + } + if (HasQuantizationInfo(info.weightTensor)) { + info.weightQuant = GetQuantizationInfo(info.weightTensor); + CheckQuantInfo(info.weightQuant, "weight", true, reasons); + } else { + reasons.push_back("weight tensor has no QuantizationInfo"); + } + } + + if (!info.biasTensor.empty()) { + if (auto producer = requireQuantProducer(info.biasTensor, "bias")) { + info.biasQuantOpIndex = *producer; + info.biasSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); + } + if (HasQuantizationInfo(info.biasTensor)) { + info.biasQuant = GetQuantizationInfo(info.biasTensor); + CheckQuantInfo(*info.biasQuant, "bias", true, reasons); + } else { + reasons.push_back("bias tensor has no QuantizationInfo"); + } + } + + if (!info.gemmOutputTensor.empty()) { + auto consumers = consumersByTensor.find(info.gemmOutputTensor); + if (consumers == consumersByTensor.end() || consumers->second.empty()) { + reasons.push_back("Gemm output has no output quantization consumer"); + } else if (consumers->second.size() != 1) { + reasons.push_back("Gemm output has multiple consumers"); + } else { + auto consumerIndex = consumers->second.front(); + if (!fOperators[consumerIndex]->IsQuantizationBoundary()) { + reasons.push_back("Gemm output consumer is not a quantization boundary"); + } else { + info.outputQuantOpIndex = consumerIndex; + auto quantOutputs = fOperators[consumerIndex]->GetOpOutputTensors(); + if (quantOutputs.size() != 1) { + reasons.push_back("output quantization boundary does not have exactly one output"); + } else { + info.outputTensor = std::string(quantOutputs[0]); + if (HasQuantizationInfo(info.outputTensor)) { + info.outputQuant = GetQuantizationInfo(info.outputTensor); + CheckQuantInfo(info.outputQuant, "output", false, reasons); + } else { + reasons.push_back("output tensor has no QuantizationInfo"); + } + } + } + } + } + + const bool hasQuantizationEvidence = + info.inputQuantOpIndex != static_cast(-1) || + info.weightQuantOpIndex != static_cast(-1) || + info.biasQuantOpIndex.has_value() || info.outputQuantOpIndex != static_cast(-1) || + (!info.inputTensor.empty() && HasQuantizationInfo(info.inputTensor)) || + (!info.weightTensor.empty() && HasQuantizationInfo(info.weightTensor)) || + (!info.biasTensor.empty() && HasQuantizationInfo(info.biasTensor)) || + (!info.outputTensor.empty() && HasQuantizationInfo(info.outputTensor)); + + if (reasons.empty()) { + info.status = EQuantizedLoweringStatus::SemanticRecognized; + info.reason = "recognized quantized Gemm region"; + auto cpuPlan = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::CPU, "CPU quantized Gemm lowering requires constant pre-quantized weight storage", true); + auto alpakaPlan = MakeAlpakaFakeQuantPlan(info); + + if (!info.weightSourceTensor.empty() && IsInitializedTensor(info.weightSourceTensor)) { + constexpr std::size_t packedTileN = 4; + const auto storageTensor = info.weightSourceTensor + "_s11_packed_cpu_storage"; + const auto weightShape = GetTensorShape(info.weightSourceTensor); + if (weightShape.size() != 2) { + reasons.push_back("weight tensor is not rank-2 for packed CPU storage"); + } else { + const auto n = weightShape[0]; + const auto k = weightShape[1]; + const auto packedBlocks = (n + packedTileN - 1) / packedTileN; + const std::vector packedShape = { packedBlocks, k, packedTileN }; + const auto packedLength = ConvertShapeToLength(packedShape); + const float *weightData = fInitializedTensors.at(info.weightSourceTensor).data(); + + if (info.weightQuant.isSigned) { + std::vector packedWeights(packedLength, 0); + for (std::size_t block = 0; block < packedBlocks; ++block) { + for (std::size_t kk = 0; kk < k; ++kk) { + for (std::size_t ji = 0; ji < packedTileN; ++ji) { + const auto col = block * packedTileN + ji; + if (col < n) { + packedWeights[(block * k + kk) * packedTileN + ji] = + static_cast(QuantizeScalarToIntegerGrid(weightData[col * k + kk], info.weightQuant)); + } + } + } + } + if (!IsInitializedTensor(storageTensor)) { + AddConstantTensor(storageTensor, packedShape, packedWeights); + } + } else { + std::vector packedWeights(packedLength, 0); + for (std::size_t block = 0; block < packedBlocks; ++block) { + for (std::size_t kk = 0; kk < k; ++kk) { + for (std::size_t ji = 0; ji < packedTileN; ++ji) { + const auto col = block * packedTileN + ji; + if (col < n) { + packedWeights[(block * k + kk) * packedTileN + ji] = + static_cast(QuantizeScalarToIntegerGrid(weightData[col * k + kk], info.weightQuant)); + } + } + } + } + if (!IsInitializedTensor(storageTensor)) { + AddConstantTensor(storageTensor, packedShape, packedWeights); + } + } + + QuantizedTensorStorage storage; + storage.logicalTensor = info.weightTensor; + storage.sourceTensor = info.weightSourceTensor; + storage.storageTensor = storageTensor; + storage.storageType = StorageTypeForQuantizedTensor(info.weightQuant); + storage.layout = EQuantizedLayout::PackedCPU; + storage.quantization = info.weightQuant; + storage.shape = packedShape; + storage.isConstant = true; + storage.isPersistent = true; + storage.isTransient = false; + storage.isDeviceResident = false; + storage.byteSize = QuantizedStorageByteSize(storage.storageType, storage.shape); + RegisterQuantizedTensorStorage(std::move(storage)); + + cpuPlan = MakeCPUPackedWeightBaselinePlan(info, storageTensor); + } + } + + auto &plans = fQuantizationState.loweringPlans[opIndex]; + plans[EQuantizedBackend::CPU] = std::move(cpuPlan); + plans[EQuantizedBackend::ALPAKA] = std::move(alpakaPlan); + fQuantizationState.gemmRegions[opIndex] = std::move(info); + } else { + info.reason = JoinReasons(reasons); + if (hasQuantizationEvidence) { + auto &plans = fQuantizationState.loweringPlans[opIndex]; + plans[EQuantizedBackend::CPU] = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::CPU, info.reason, true); + plans[EQuantizedBackend::ALPAKA] = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::ALPAKA, info.reason, true); + fQuantizationState.gemmRegions[opIndex] = std::move(info); + } + if (fVerbose > 0) { + std::cout << "SOFIE quantized Gemm candidate rejected at operator " << opIndex << ": " << info.reason << std::endl; + } + } + } +} + +void RModel::AddLoweredQuantizedGemmOperators(EQuantizedBackend backend) +{ + for (auto op_idx : GetQuantizedGemmOperatorIndices()) { + if (!HasQuantizedLoweringPlan(op_idx, backend)) + continue; + + auto *gemm = dynamic_cast *>(fOperators[op_idx].get()); + if (!gemm) + throw std::runtime_error("SOFIE quantized Gemm region is attached to a non-float Gemm operator"); + + const auto &plan = GetQuantizedLoweringPlan(op_idx, backend); + if (!IsQuantizedLoweringAvailable(plan.status)) + continue; + + fLoweredOperators[op_idx] = std::make_unique( + GetQuantizedGemmRegion(op_idx), plan, MakeQuantizedGemmCodegenContext(*gemm)); + + if (plan.suppressesGraphOperators) { + for (auto consumedOpIndex : plan.consumedOperatorIndices) { + if (consumedOpIndex != op_idx) + fLoweredConsumedOperatorIndices.insert(consumedOpIndex); + } + } + } +} + +void RModel::AddQuantizedGeneratedHeaders() +{ + for (auto op_idx : GetQuantizedGemmOperatorIndices()) { + if (!HasQuantizedLoweringPlan(op_idx, EQuantizedBackend::CPU)) + continue; + const auto &plan = GetQuantizedLoweringPlan(op_idx, EQuantizedBackend::CPU); + if (IsQuantizedLoweringAvailable(plan.status) && plan.suppressesGraphOperators && + plan.usesPrequantizedWeights && plan.weightLayout == EQuantizedLayout::PackedCPU) { + AddNeededCustomHeader("SOFIE/SOFIE_QuantizedRuntime.hxx"); + } + } +} + +} // namespace SOFIE diff --git a/parsers/CMakeLists.txt b/parsers/CMakeLists.txt index 7174e90..af87659 100644 --- a/parsers/CMakeLists.txt +++ b/parsers/CMakeLists.txt @@ -41,6 +41,7 @@ set(sources_cxx src/ParseConv.cxx src/ParseConvTranspose.cxx src/ParseGemm.cxx + src/ParseQONNXQuant.cxx src/ParseGRU.cxx src/ParseIdentity.cxx src/ParseLeakyRelu.cxx diff --git a/parsers/inc/SOFIE/RModelParser_ONNX.hxx b/parsers/inc/SOFIE/RModelParser_ONNX.hxx index c1dc463..93d6fa1 100644 --- a/parsers/inc/SOFIE/RModelParser_ONNX.hxx +++ b/parsers/inc/SOFIE/RModelParser_ONNX.hxx @@ -35,17 +35,24 @@ private: std::unique_ptr fOperatorsMapImpl; // Type of the tensors std::unordered_map fTensorTypeMap; + std::unordered_map fOpsetVersionMap; // flag list of fused operators std::vector fFusedOperators; public: - // Register an ONNX operator + // Register an ONNX operator in the standard/default ONNX domain void RegisterOperator(const std::string &name, ParserFuncSignature func); - // Check if the operator is registered + // Register an ONNX operator in an explicit domain. The standard ONNX domain is "". + void RegisterOperator(const std::string &domain, const std::string &name, ParserFuncSignature func); + + // Check if the operator is registered in the standard/default ONNX domain bool IsRegisteredOperator(const std::string &name); + // Check if the operator is registered in an explicit domain + bool IsRegisteredOperator(const std::string &domain, const std::string &name); + // List of registered operators (in alphabetical order) std::vector GetRegisteredOperators(); @@ -63,6 +70,9 @@ public: // Get the type of the tensor ETensorType GetTensorType(const std::string &name); + // Get imported ONNX opset version for a domain. Returns -1 if absent. + int GetOpsetVersion(const std::string &domain) const; + // Parse the index'th node from the ONNX graph std::unique_ptr ParseOperator(const size_t /*index*/, const onnx::GraphProto & /*graphproto*/, const std::vector & /*nodes*/, const std::vector & /* children */); diff --git a/parsers/src/ParseQONNXQuant.cxx b/parsers/src/ParseQONNXQuant.cxx new file mode 100644 index 0000000..1fd5029 --- /dev/null +++ b/parsers/src/ParseQONNXQuant.cxx @@ -0,0 +1,88 @@ +#include "SOFIE/RModelParser_ONNX.hxx" +#include "SOFIE/ROperator_QONNXQuant.hxx" +#include "onnx_proto3.pb.h" + +#include +#include + +namespace SOFIE { + +namespace { + +EQuantizationRoundingMode ParseQONNXRoundingMode(const std::string &roundingMode) +{ + if (roundingMode == "ROUND") return EQuantizationRoundingMode::ROUND; + if (roundingMode == "FLOOR") return EQuantizationRoundingMode::FLOOR; + if (roundingMode == "TRUNCATE") return EQuantizationRoundingMode::TRUNCATE; + throw std::runtime_error("TMVA::SOFIE QONNX Quant unsupported rounding_mode " + roundingMode); +} + +} // namespace + +ParserFuncSignature ParseQONNXQuant = [](RModelParser_ONNX &parser, const onnx::NodeProto &nodeproto) { + if (nodeproto.domain() != "qonnx.custom_op.general") { + throw std::runtime_error("TMVA::SOFIE QONNX Quant parser received unexpected domain " + nodeproto.domain()); + } + if (parser.GetOpsetVersion("qonnx.custom_op.general") != 1) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant requires qonnx.custom_op.general opset version 1"); + } + if (nodeproto.input_size() != 4) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant expects four inputs: tensor, scale, zero_point, bit_width"); + } + if (nodeproto.output_size() != 1) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant expects one output"); + } + + const std::string inputName = nodeproto.input(0); + const std::string scaleName = nodeproto.input(1); + const std::string zeroPointName = nodeproto.input(2); + const std::string bitWidthName = nodeproto.input(3); + const std::string outputName = nodeproto.output(0); + + if (!parser.IsRegisteredTensorType(inputName)) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant input tensor " + inputName + " has no registered type"); + } + if (!parser.IsRegisteredTensorType(scaleName) || !parser.IsRegisteredTensorType(zeroPointName) || + !parser.IsRegisteredTensorType(bitWidthName)) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant scale, zero_point, and bit_width must be registered tensors"); + } + + bool isSigned = false; + bool narrow = false; + bool hasSigned = false; + bool hasRoundingMode = false; + EQuantizationRoundingMode rounding = EQuantizationRoundingMode::UNDEFINED; + + for (const auto &attribute : nodeproto.attribute()) { + if (attribute.name() == "signed") { + isSigned = attribute.i() != 0; + hasSigned = true; + } else if (attribute.name() == "narrow") { + narrow = attribute.i() != 0; + } else if (attribute.name() == "rounding_mode") { + rounding = ParseQONNXRoundingMode(attribute.s()); + hasRoundingMode = true; + } else { + throw std::runtime_error("TMVA::SOFIE QONNX Quant unsupported attribute " + attribute.name()); + } + } + + if (!hasSigned) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant missing required signed attribute"); + } + if (!hasRoundingMode) { + throw std::runtime_error("TMVA::SOFIE QONNX Quant missing required rounding_mode attribute"); + } + + // The initial PQuant/QONNX profile accepts saturating per-tensor affine semantics only. + const auto overflow = narrow ? EQuantizationOverflowMode::SAT_SYM : EQuantizationOverflowMode::SAT; + + if (!parser.IsRegisteredTensorType(outputName)) { + parser.RegisterTensorType(outputName, parser.GetTensorType(inputName)); + } + + return std::make_unique(inputName, scaleName, zeroPointName, bitWidthName, outputName, + isSigned, narrow, rounding, overflow); +}; + +} // namespace SOFIE diff --git a/parsers/src/RModelParser_ONNX.cxx b/parsers/src/RModelParser_ONNX.cxx index afb8b93..20a8eb5 100644 --- a/parsers/src/RModelParser_ONNX.cxx +++ b/parsers/src/RModelParser_ONNX.cxx @@ -73,6 +73,7 @@ extern ParserFuncSignature ParseLeakyRelu; extern ParserFuncSignature ParseSelu; extern ParserFuncSignature ParseSigmoid; extern ParserFuncSignature ParseGemm; +extern ParserFuncSignature ParseQONNXQuant; extern ParserFuncSignature ParseRNN; extern ParserFuncSignature ParseLSTM; extern ParserFuncSignature ParsePool; @@ -117,10 +118,61 @@ extern ParserFuseFuncSignature ParseFuseBatchnormRelu; extern ParserFuseFuncSignature ParseFuseConvTransposeAdd; extern ParserFuseFuncSignature ParseFuseMatMulAdd; +namespace { + +std::string NormalizeONNXDomain(const std::string &domain) +{ + // ONNX's default operator set is normally encoded with an empty domain. + // Just treat the explicit ai.onnx spelling as the same standard domain if it appears. + return (domain.empty() || domain == "ai.onnx") ? std::string{} : domain; +} + +std::string FormatQualifiedOperatorName(const std::string &domain, const std::string &opType) +{ + const std::string normalizedDomain = NormalizeONNXDomain(domain); + return normalizedDomain.empty() ? opType : normalizedDomain + "::" + opType; +} + + +void PopulateOpsetVersionMap(const onnx::ModelProto &model, std::unordered_map &opsetVersionMap) +{ + opsetVersionMap.clear(); + for (int i = 0; i < model.opset_import_size(); ++i) { + const auto &opset = model.opset_import(i); + opsetVersionMap[NormalizeONNXDomain(opset.domain())] = static_cast(opset.version()); + } +} + +} // namespace + // Definition of RModelParser_ONNX::OperatorsMap struct RModelParser_ONNX::OperatorsMapImpl { - // Registered operators - std::unordered_map fOperatorsMap; + struct OperatorKey { + std::string fDomain; + std::string fOpType; + + bool operator==(const OperatorKey &other) const + { + return fDomain == other.fDomain && fOpType == other.fOpType; + } + }; + + struct OperatorKeyHash { + std::size_t operator()(const OperatorKey &key) const + { + const std::size_t h1 = std::hash{}(key.fDomain); + const std::size_t h2 = std::hash{}(key.fOpType); + return h1 ^ (h2 + 0x9e3779b97f4a7c15ULL + (h1 << 6) + (h1 >> 2)); + } + }; + + static OperatorKey MakeKey(const std::string &domain, const std::string &opType) + { + return {NormalizeONNXDomain(domain), opType}; + } + + // Registered operators keyed by ONNX domain and operator type. + std::unordered_map fOperatorsMap; }; // helper function to get initialized tensor data @@ -241,6 +293,7 @@ RModelParser_ONNX::RModelParser_ONNX() noexcept : fOperatorsMapImpl(std::make_un RegisterOperator("Conv", ParseConv); RegisterOperator("ConvTranspose", ParseConvTranspose); RegisterOperator("Gemm", ParseGemm); + RegisterOperator("qonnx.custom_op.general", "Quant", ParseQONNXQuant); RegisterOperator("GRU", ParseGRU); RegisterOperator("Identity", ParseIdentity); RegisterOperator("LeakyRelu", ParseLeakyRelu); @@ -299,12 +352,23 @@ RModelParser_ONNX::~RModelParser_ONNX() = default; void RModelParser_ONNX::RegisterOperator(const std::string &name, ParserFuncSignature func) { - fOperatorsMapImpl->fOperatorsMap[name] = func; + RegisterOperator("", name, std::move(func)); +} + +void RModelParser_ONNX::RegisterOperator(const std::string &domain, const std::string &name, ParserFuncSignature func) +{ + fOperatorsMapImpl->fOperatorsMap[RModelParser_ONNX::OperatorsMapImpl::MakeKey(domain, name)] = std::move(func); } bool RModelParser_ONNX::IsRegisteredOperator(const std::string &name) { - return fOperatorsMapImpl->fOperatorsMap.find(name) != fOperatorsMapImpl->fOperatorsMap.end(); + return IsRegisteredOperator("", name); +} + +bool RModelParser_ONNX::IsRegisteredOperator(const std::string &domain, const std::string &name) +{ + return fOperatorsMapImpl->fOperatorsMap.find(RModelParser_ONNX::OperatorsMapImpl::MakeKey(domain, name)) != + fOperatorsMapImpl->fOperatorsMap.end(); } std::vector RModelParser_ONNX::GetRegisteredOperators() @@ -312,7 +376,7 @@ std::vector RModelParser_ONNX::GetRegisteredOperators() std::vector ops; ops.reserve(fOperatorsMapImpl->fOperatorsMap.size()); for (auto &it : fOperatorsMapImpl->fOperatorsMap) { - ops.emplace_back(it.first); + ops.emplace_back(FormatQualifiedOperatorName(it.first.fDomain, it.first.fOpType)); } // return sorted list in alphabetical order std::sort(ops.begin(), ops.end()); @@ -334,6 +398,12 @@ ETensorType RModelParser_ONNX::GetTensorType(const std::string &name) return fTensorTypeMap[UTILITY::Clean_name(name)]; } +int RModelParser_ONNX::GetOpsetVersion(const std::string &domain) const +{ + auto it = fOpsetVersionMap.find(NormalizeONNXDomain(domain)); + return it == fOpsetVersionMap.end() ? -1 : it->second; +} + // Parse an operator std::unique_ptr RModelParser_ONNX::ParseOperator(const size_t i, const onnx::GraphProto &graphproto, const std::vector &nodes, const std::vector & children) @@ -343,14 +413,16 @@ RModelParser_ONNX::ParseOperator(const size_t i, const onnx::GraphProto &graphpr int idx = nodes[i]; const auto &nodeproto = graphproto.node(idx); const std::string op_type = nodeproto.op_type(); + const std::string op_domain = NormalizeONNXDomain(nodeproto.domain()); if (fVerbose) - std::cout << "Parsing operator " << op_type << std::endl; + std::cout << "Parsing operator " << FormatQualifiedOperatorName(op_domain, op_type) << std::endl; // skip already fused operators if (fFusedOperators[idx]) return nullptr; - // try to fuse with following operator in case it is not last one - if (children.size() == 1) { + // try to fuse with following operator in case it is not last one. + // Parser fusions are defined only for standard ONNX operators. + if (op_domain.empty() && children.size() == 1) { int idx2 = children.front(); if (op_type == "MatMul") { // Fuse MatMul and Add @@ -388,13 +460,15 @@ RModelParser_ONNX::ParseOperator(const size_t i, const onnx::GraphProto &graphpr - auto it = fOperatorsMapImpl->fOperatorsMap.find(op_type); + const auto op_key = RModelParser_ONNX::OperatorsMapImpl::MakeKey(op_domain, op_type); + auto it = fOperatorsMapImpl->fOperatorsMap.find(op_key); if (it == fOperatorsMapImpl->fOperatorsMap.end()) { - std::cout << "operator " << op_type << " is not supported" << std::endl; - throw std::runtime_error("TMVA::SOFIE Operator type " + op_type + " is not yet supported"); + const std::string qualifiedName = FormatQualifiedOperatorName(op_domain, op_type); + std::cout << "operator " << qualifiedName << " is not supported" << std::endl; + throw std::runtime_error("TMVA::SOFIE Operator type " + qualifiedName + " is not yet supported"); } if (fVerbose) { - std::cout << "\tCreating operator " << op_type << std::endl; + std::cout << "\tCreating operator " << FormatQualifiedOperatorName(op_domain, op_type) << std::endl; } return it->second(*this, nodeproto); } @@ -405,11 +479,14 @@ RModel RModelParser_ONNX::Parse(std::string filename, bool verbose) fVerbose = verbose; fTensorTypeMap.clear(); + fOpsetVersionMap.clear(); auto model = LoadModel(filename); if (!model) throw std::runtime_error("TMVA::SOFIE - Failed to load onnx file " + filename); + PopulateOpsetVersionMap(*model, fOpsetVersionMap); + const onnx::GraphProto &graph = model->graph(); // not a memory leak. model freed automatically at the end. @@ -459,8 +536,10 @@ void RModelParser_ONNX::CheckGraph(const onnx::GraphProto & graph, int & level, for (int i = 0; i < graph.node_size(); i++) { const auto & node = graph.node(i); const std::string opType = node.op_type(); + const std::string opDomain = NormalizeONNXDomain(node.domain()); + const std::string qualifiedName = FormatQualifiedOperatorName(opDomain, opType); if (fVerbose) { - std::cout << "\tOperator " << i << " : " << opType << " (" << node.name() << "), " << graph.node(i).input_size() + std::cout << "\tOperator " << i << " : " << qualifiedName << " (" << node.name() << "), " << graph.node(i).input_size() << " inputs : {"; for (int j = 0; j < graph.node(i).input_size(); j++) { std::cout << graph.node(i).input(j); @@ -470,8 +549,8 @@ void RModelParser_ONNX::CheckGraph(const onnx::GraphProto & graph, int & level, std::cout << " }" << std::endl; } // check if operator exists - if (!IsRegisteredOperator(opType)) - missingOperators[opType] = level; + if (!IsRegisteredOperator(opDomain, opType)) + missingOperators[qualifiedName] = level; // see if sub-graph exists as node attributes for (int j = 0; j < node.attribute_size(); j++) { const auto & attribute = node.attribute(j); @@ -487,9 +566,12 @@ void RModelParser_ONNX::CheckGraph(const onnx::GraphProto & graph, int & level, bool RModelParser_ONNX::CheckModel(std::string filename, bool verbose) { fVerbose = verbose; + fOpsetVersionMap.clear(); auto model = LoadModel(filename); if (!model) return false; + PopulateOpsetVersionMap(*model, fOpsetVersionMap); + const onnx::GraphProto &graph = model->graph(); // Initial operator order if (fVerbose) From a1625f70ad035e4cedc965b0e3e136d72ccd7efd Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Thu, 2 Jul 2026 13:35:26 +0200 Subject: [PATCH 2/6] feat: add QONNX quantized GEMM GPU lowering with cuBLASLt --- core/CMakeLists.txt | 3 +- core/inc/SOFIE/RModel.hxx | 12 +- core/inc/SOFIE/ROperator_QuantizedGemm.hxx | 174 +++- core/inc/SOFIE/RQuantization.hxx | 91 +- ...antizedRuntime.hxx => SOFIE_Quantized.hxx} | 54 +- core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx | 918 ++++++++++++++++++ core/src/RModel.cxx | 4 +- core/src/RModel_ALPAKA.cxx | 137 ++- core/src/RModel_Quantization.cxx | 449 ++++++--- 9 files changed, 1606 insertions(+), 236 deletions(-) rename core/inc/SOFIE/{SOFIE_QuantizedRuntime.hxx => SOFIE_Quantized.hxx} (87%) create mode 100644 core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 99d3d86..d474027 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -28,7 +28,8 @@ set(sources_headers SOFIE/ROperator_ConvTranspose.hxx SOFIE/ROperator_Gemm.hxx SOFIE/ROperator_QuantizedGemm.hxx - SOFIE/SOFIE_QuantizedRuntime.hxx + SOFIE/SOFIE_Quantized.hxx + SOFIE/SOFIE_QuantizedAlpaka.hxx SOFIE/RQuantization.hxx SOFIE/ROperator_Relu.hxx SOFIE/ROperator_Tanh.hxx diff --git a/core/inc/SOFIE/RModel.hxx b/core/inc/SOFIE/RModel.hxx index b7943f7..897277a 100644 --- a/core/inc/SOFIE/RModel.hxx +++ b/core/inc/SOFIE/RModel.hxx @@ -82,16 +82,8 @@ private: /// handled by the ONNX parser) into a single in-place kernel sequence. void FuseGemmActivations_GPU(); void BuildLoweredOperatorView(EQuantizedBackend backend = EQuantizedBackend::CPU); - void AddQuantizedGeneratedHeaders(); - - // private gemm-specific methods for RModel_Quantization.cxx. must move out of - // RModel into a dedicated state once more operators are added. - void AddLoweredQuantizedGemmOperators(EQuantizedBackend backend = EQuantizedBackend::CPU); - bool HasQuantizedGemmRegion(std::size_t op_index) const; - const QuantizedGemmRegion & GetQuantizedGemmRegion(std::size_t op_index) const; - std::vector GetQuantizedGemmOperatorIndices() const; - bool HasQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const; - const QuantizedLoweringPlan & GetQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const; + void AddQuantizedGeneratedHeaders(EQuantizedBackend backend = EQuantizedBackend::CPU); + void AddLoweredQuantizedOperators(EQuantizedBackend backend = EQuantizedBackend::CPU); public: // Rule of five: explicitly define move semantics, disallow copy diff --git a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx index 9c5103a..c1ff230 100644 --- a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx +++ b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx @@ -4,7 +4,7 @@ #include "SOFIE/ROperator.hxx" #include "SOFIE/SOFIE_common.hxx" #include "SOFIE/RQuantization.hxx" -#include "SOFIE/SOFIE_QuantizedRuntime.hxx" +#include "SOFIE/SOFIE_Quantized.hxx" #include #include @@ -20,10 +20,6 @@ namespace SOFIE { struct QuantizedGemmCodegenContext { - std::string inputTensor; - std::string weightTensor; - std::string biasTensor; - std::string outputTensor; std::vector inputShape; std::vector weightShape; std::vector outputShape; @@ -32,7 +28,6 @@ struct QuantizedGemmCodegenContext { std::int64_t transA = 0; std::int64_t transB = 0; EActivationType activation = EActivationType::UNDEFINED; - std::string indent = " "; }; namespace INTERNAL { @@ -57,50 +52,41 @@ inline void ValidateQuantizedGemmContext(const QuantizedGemmCodegenContext &cont } } -inline bool NearlyEqualQuantizedScale(double lhs, double rhs) +inline bool HasQuantizedGemmBias(const QuantizedGemmRegion ®ion) { - const auto scale = std::max({1.0, std::fabs(lhs), std::fabs(rhs)}); - return std::fabs(lhs - rhs) <= (1.0e-12 * scale); + return !region.biasSourceTensor.empty(); } -inline bool MakeQuantizedGemmFixedPointMultiplier(double realMultiplier, std::int64_t &multiplier, int &shift) +inline const char *QuantizedCudaInputCarrierName(EQuantizedCarrierMode mode) { - if (!std::isfinite(realMultiplier) || realMultiplier <= 0.0) { - return false; + switch (mode) { + case EQuantizedCarrierMode::Int8: + return "Int8"; + case EQuantizedCarrierMode::Float: + return "Float"; + default: + throw std::runtime_error("SOFIE quantized CUDA GEMM received unsupported input carrier mode"); } - - int exponent = 0; - const double significand = std::frexp(realMultiplier, &exponent); - auto q31 = static_cast(std::llround(significand * 2147483648.0)); - if (q31 == (std::int64_t{1} << 31)) { - q31 /= 2; - ++exponent; - } - - const int candidateShift = 31 - exponent; - if (q31 <= 0 || candidateShift < 0 || candidateShift >= 62) { - return false; - } - - multiplier = q31; - shift = candidateShift; - return true; } -inline bool MakeExactIntegerScaleMultiplier(double scale, std::int64_t &multiplier, int &shift) +inline const char *QuantizedCudaEpilogueModeName(EQuantizedOutputMode mode) { - if (!std::isfinite(scale) || scale < 0.0) { - return false; + switch (mode) { + case EQuantizedOutputMode::ExactFakeQuantFloat: + return "ExactFakeQuant"; + case EQuantizedOutputMode::Quantized: + return "Quantized"; + default: + throw std::runtime_error("SOFIE quantized CUDA GEMM received unsupported output mode"); } +} - const auto rounded = std::llround(scale); - if (!NearlyEqualQuantizedScale(scale, static_cast(rounded))) { - return false; +inline const char *QuantizedCudaOutputCarrierName(const QuantizationInfo &info) +{ + if (info.bitWidth != 8) { + throw std::runtime_error("SOFIE quantized CUDA GEMM currently supports only 8-bit output carriers"); } - - multiplier = rounded; - shift = 0; - return true; + return info.isSigned ? "Int8" : "UInt8"; } } // namespace INTERNAL @@ -112,7 +98,7 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con opName = "op_" + opName; INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm call CPU"); - if (!plan.usesPrequantizedWeights) { + if (!QuantizedPlanUsesPrequantizedWeights(plan)) { throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path requires pre-quantized weight storage"); } if (plan.weightStorageTensor.empty()) { @@ -127,7 +113,7 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con if (region.inputSourceTensor.empty() || region.outputTensor.empty()) { throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path is missing quantization source/output tensors"); } - if (!context.biasTensor.empty() && (region.biasSourceTensor.empty() || !region.biasQuant.has_value())) { + if (INTERNAL::HasQuantizedGemmBias(region) && !region.biasQuant.has_value()) { throw std::runtime_error("SOFIE fused Quantized Gemm call CPU path is missing quantized bias metadata"); } @@ -136,7 +122,7 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con const auto m = context.inputShape[dimA - 2].GetVal(); const auto k = context.inputShape[dimA - 1].GetVal(); const auto n = context.weightShape[dimB - 2].GetVal(); - const auto &SP = context.indent; + const std::string SP = " "; constexpr std::size_t tileN = 4; const auto inputRange = QuantizedIntegerRange(region.inputQuant); @@ -174,17 +160,17 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con std::int64_t requantMultiplier = 0; int requantShift = 0; const bool hasFixedPointRequantization = - INTERNAL::MakeQuantizedGemmFixedPointMultiplier(requantScale, requantMultiplier, requantShift); + MakeQuantizedFixedPointMultiplier(requantScale, requantMultiplier, requantShift); - const bool hasBiasTensor = !context.biasTensor.empty(); + const bool hasBiasTensor = INTERNAL::HasQuantizedGemmBias(region); const bool hasAccumulatorBias = !hasBiasTensor || - (region.biasQuant.has_value() && INTERNAL::NearlyEqualQuantizedScale(region.biasQuant->scale, accumulatorScale)); + (region.biasQuant.has_value() && NearlyEqualQuantizedScale(region.biasQuant->scale, accumulatorScale)); std::int64_t biasRequantMultiplier = 0; int biasRequantShift = 0; const bool hasExactIntegerBias = !hasBiasTensor || - (region.biasQuant.has_value() && INTERNAL::MakeExactIntegerScaleMultiplier( + (region.biasQuant.has_value() && MakeExactIntegerScaleMultiplier( region.biasQuant->scale / region.outputQuant.scale, biasRequantMultiplier, biasRequantShift)); const bool useIntegerEpilogue = hasFixedPointRequantization && (!hasBiasTensor || hasAccumulatorBias || hasExactIntegerBias); @@ -216,7 +202,7 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con out << SP << opName << "_params.activation = SOFIE::EQuantizedGemmActivation::None;\n"; } - if (!context.biasTensor.empty()) { + if (INTERNAL::HasQuantizedGemmBias(region)) { const auto biasRange = QuantizedIntegerRange(*region.biasQuant); out << SP << opName << "_params.hasBias = true;\n"; out << SP << opName << "_params.scaleB = " @@ -230,7 +216,7 @@ inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, con out << SP << "SOFIE::QuantizedGemm_Call(tensor_" << region.outputTensor << ", tensor_" << region.inputSourceTensor << ", tensor_" << plan.weightStorageTensor << ", "; - if (!context.biasTensor.empty()) { + if (INTERNAL::HasQuantizedGemmBias(region)) { out << "tensor_" << region.biasSourceTensor; } else { out << "nullptr"; @@ -294,6 +280,84 @@ inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantDefinition(std::stri return " QuantizedGemmAlpakaFakeQuantKernel_" + opName + " quantizedGemmAlpakaFakeQuantKernel_" + opName + ";\n"; } +inline std::string GenerateFusedQuantizedGemmCublasLtCoreLaunch(std::string opName, + const QuantizedGemmCodegenContext &context, + const QuantizedGemmRegion ®ion, + const QuantizedLoweringPlan &plan) +{ + INTERNAL::ValidateQuantizedGemmContext(context, "fused Quantized Gemm cuBLASLt core launch"); + if (plan.backend != EQuantizedBackend::ALPAKA || plan.status != EQuantizedLoweringStatus::Optimized || + !QuantizedPlanUsesPrequantizedWeights(plan) || plan.weightLayout != EQuantizedLayout::PlainDevice || + plan.weightStorageTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm cuBLASLt core launch requires an optimized Alpaka PlainDevice plan"); + } + if (region.inputSourceTensor.empty() || region.outputTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm cuBLASLt core launch is missing input/output tensors"); + } + + const auto dimA = context.inputShape.size(); + const auto dimB = context.weightShape.size(); + const auto m = context.inputShape[dimA - 2].GetVal(); + const auto k = context.inputShape[dimA - 1].GetVal(); + const auto n = context.weightShape[dimB - 2].GetVal(); + + std::stringstream out; + out << "\n//--------- ROperator_QuantizedGemm cuBLASLt int8 GEMM core boundary " << opName << "\n"; + out << " // Optimized GPU boundary: stream-ordered cuBLASLt int8 GEMM selected by the lowering plan.\n"; + out << " {\n"; + out << " // Quantized lowering capability: " << plan.capabilityTag << "\n"; + out << " // Quantized lowering reason: " << plan.reason << "\n"; + out << " SOFIE::QuantizedGemmCudaLtParams params_quantizedGemm_" << opName << "{};\n"; + out << " params_quantizedGemm_" << opName << ".m = static_cast(" << m << ");\n"; + out << " params_quantizedGemm_" << opName << ".n = static_cast(" << n << ");\n"; + out << " params_quantizedGemm_" << opName << ".k = static_cast(" << k << ");\n"; + out << std::setprecision(17); + out << " params_quantizedGemm_" << opName << ".inputScale = " << region.inputQuant.scale << ";\n"; + out << " params_quantizedGemm_" << opName << ".weightScale = " << region.weightQuant.scale << ";\n"; + out << " params_quantizedGemm_" << opName << ".biasScale = " << (region.biasQuant ? region.biasQuant->scale : 1.0) << ";\n"; + out << " params_quantizedGemm_" << opName << ".outputScale = " << region.outputQuant.scale << ";\n"; + out << " params_quantizedGemm_" << opName << ".inputZeroPoint = " << region.inputQuant.zeroPoint << ";\n"; + out << " params_quantizedGemm_" << opName << ".weightZeroPoint = " << region.weightQuant.zeroPoint << ";\n"; + out << " params_quantizedGemm_" << opName << ".biasZeroPoint = " << (region.biasQuant ? region.biasQuant->zeroPoint : 0) << ";\n"; + out << " params_quantizedGemm_" << opName << ".outputZeroPoint = " << region.outputQuant.zeroPoint << ";\n"; + const auto inputRange = QuantizedIntegerRange(region.inputQuant); + const auto biasRange = region.biasQuant ? QuantizedIntegerRange(*region.biasQuant) : std::pair{0, 0}; + const auto outputRange = QuantizedIntegerRange(region.outputQuant); + out << " params_quantizedGemm_" << opName << ".inputQMin = static_cast(" << inputRange.first << ");\n"; + out << " params_quantizedGemm_" << opName << ".inputQMax = static_cast(" << inputRange.second << ");\n"; + out << " params_quantizedGemm_" << opName << ".biasQMin = static_cast(" << biasRange.first << ");\n"; + out << " params_quantizedGemm_" << opName << ".biasQMax = static_cast(" << biasRange.second << ");\n"; + out << " params_quantizedGemm_" << opName << ".outputQMin = static_cast(" << outputRange.first << ");\n"; + out << " params_quantizedGemm_" << opName << ".outputQMax = static_cast(" << outputRange.second << ");\n"; + out << " params_quantizedGemm_" << opName << ".hasBias = " << (INTERNAL::HasQuantizedGemmBias(region) ? "true" : "false") << ";\n"; + out << " params_quantizedGemm_" << opName << ".hasRelu = " << (context.activation == EActivationType::RELU ? "true" : "false") << ";\n"; + out << " params_quantizedGemm_" << opName << ".maxWorkspaceBytes = 32ULL * 1024ULL * 1024ULL;\n"; + const auto planEpilogueMode = INTERNAL::QuantizedCudaEpilogueModeName(plan.outputMode); + const auto planInputCarrier = INTERNAL::QuantizedCudaInputCarrierName(plan.inputCarrierMode); + out << " params_quantizedGemm_" << opName << ".epilogueMode = SOFIE::EQuantizedCudaEpilogueMode::" << planEpilogueMode << ";\n"; + out << " params_quantizedGemm_" << opName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::" << planInputCarrier << ";\n"; + out << " params_quantizedGemm_" << opName << ".outputCarrier = SOFIE::EQuantizedCudaOutputCarrier::" << INTERNAL::QuantizedCudaOutputCarrierName(region.outputQuant) << ";\n"; + if (plan.supportsPrequantizedInputCarrier) { + out << "#ifdef SOFIE_QUANTIZED_GEMM_PREQUANTIZED_INPUT\n"; + out << " params_quantizedGemm_" << opName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::Int8;\n"; + out << "#endif\n"; + } + out << " params_quantizedGemm_" << opName << ".weightType = SOFIE::EQuantizedCudaWeightType::" << (region.weightQuant.isSigned ? "Int8" : "UInt8") << ";\n"; + out << " SOFIE::QuantizedGemmCudaLt_Call(quantizedGemmCudaLtState_" << opName + << ", alpaka::getNativeHandle(queue)" + << ", alpaka::getPtrNative(deviceBuf_" << region.outputTensor << ")" + << ", alpaka::getPtrNative(deviceBuf_" << region.inputSourceTensor << ")" + << ", alpaka::getPtrNative(deviceBuf_" << plan.weightStorageTensor << ")"; + if (INTERNAL::HasQuantizedGemmBias(region)) { + out << ", alpaka::getPtrNative(deviceBuf_" << region.biasSourceTensor << ")"; + } else { + out << ", static_cast(nullptr)"; + } + out << ", params_quantizedGemm_" << opName << ");\n"; + out << " }\n"; + return out.str(); +} + inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string opName, const QuantizedGemmCodegenContext &context, const QuantizedGemmRegion ®ion, @@ -306,7 +370,7 @@ inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string o if (region.inputSourceTensor.empty() || region.weightSourceTensor.empty() || region.outputTensor.empty()) { throw std::runtime_error("SOFIE fused Quantized Gemm Alpaka fake-quant launch is missing quantization source/output tensors"); } - if (!context.biasTensor.empty() && (region.biasSourceTensor.empty() || !region.biasQuant.has_value())) { + if (INTERNAL::HasQuantizedGemmBias(region) && !region.biasQuant.has_value()) { throw std::runtime_error("SOFIE fused Quantized Gemm Alpaka fake-quant launch is missing quantized bias metadata"); } @@ -331,7 +395,7 @@ inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string o << ", quantizedGemmAlpakaFakeQuantKernel_" << opName << ", alpaka::getPtrNative(deviceBuf_" << region.inputSourceTensor << ")" << ", alpaka::getPtrNative(deviceBuf_" << region.weightSourceTensor << ")"; - if (!context.biasTensor.empty()) { + if (INTERNAL::HasQuantizedGemmBias(region)) { out << ", alpaka::getPtrNative(deviceBuf_" << region.biasSourceTensor << ")"; } else { out << ", static_cast(nullptr)"; @@ -340,7 +404,7 @@ inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string o << ", static_cast(" << elements << ")" << ", static_cast(" << n << ")" << ", static_cast(" << k << ")" - << ", " << (!context.biasTensor.empty() ? "true" : "false") + << ", " << (INTERNAL::HasQuantizedGemmBias(region) ? "true" : "false") << ", " << (context.activation == EActivationType::RELU ? "true" : "false") << ", " << std::setprecision(std::numeric_limits::max_digits10) << region.inputQuant.scale << ", " << std::setprecision(std::numeric_limits::max_digits10) << region.weightQuant.scale @@ -389,7 +453,7 @@ public: std::string Generate(std::string opName) override { if (fPlan.backend != EQuantizedBackend::CPU || !IsQuantizedLoweringAvailable(fPlan.status) || - !fPlan.suppressesGraphOperators || !fPlan.usesPrequantizedWeights || + !fPlan.suppressesGraphOperators || !QuantizedPlanUsesPrequantizedWeights(fPlan) || fPlan.weightLayout != EQuantizedLayout::PackedCPU) { throw std::runtime_error("SOFIE ROperator_QuantizedGemm CPU code generation requires an available packed-weight CPU lowering plan"); } @@ -399,16 +463,22 @@ public: std::string Generate_GPU_Kernel_ALPAKA(std::string opName) override { + if (IsOptimizedQuantizedAlpakaPlainDevicePlan(fPlan)) + return ""; return GenerateFusedQuantizedGemmAlpakaFakeQuantKernel(std::move(opName), fContext); } std::string Generate_GPU_Kernel_Definitions_ALPAKA(std::string opName) override { + if (IsOptimizedQuantizedAlpakaPlainDevicePlan(fPlan)) + return " SOFIE::QuantizedGemmCudaLtState quantizedGemmCudaLtState_" + opName + "; // owns cuBLASLt state and CUDA temporaries\n"; return GenerateFusedQuantizedGemmAlpakaFakeQuantDefinition(std::move(opName), fContext); } std::string Generate_GPU_ALPAKA(std::string opName) override { + if (IsOptimizedQuantizedAlpakaPlainDevicePlan(fPlan)) + return GenerateFusedQuantizedGemmCublasLtCoreLaunch(std::move(opName), fContext, fRegion, fPlan); return GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::move(opName), fContext, fRegion, fPlan); } }; diff --git a/core/inc/SOFIE/RQuantization.hxx b/core/inc/SOFIE/RQuantization.hxx index 3d8667a..1398b6b 100644 --- a/core/inc/SOFIE/RQuantization.hxx +++ b/core/inc/SOFIE/RQuantization.hxx @@ -83,6 +83,11 @@ inline bool IsQuantizedLoweringUnsupported(EQuantizedLoweringStatus status) status == EQuantizedLoweringStatus::SemanticUnsupported; } +inline bool IsQuantizedLoweringOptimized(EQuantizedLoweringStatus status) +{ + return status == EQuantizedLoweringStatus::Optimized; +} + enum class EQuantizedBackend { UNDEFINED = 0, CPU = 1, ALPAKA = 2 }; @@ -91,8 +96,55 @@ enum class EQuantizedStorageType { UNDEFINED = 0, FloatCarrier = 1, Int8 = 2, UInt8 = 3, Int32Accumulator = 4, MetadataOnly = 5 }; +enum class EQuantizedCarrierMode { + UNDEFINED = 0, + Float = 1, + Int8 = 2, + UInt8 = 3, + Int32Accumulator = 4 +}; + +enum class EQuantizedOutputMode { + UNDEFINED = 0, + ExactFakeQuantFloat = 1, + Quantized = 2, + Int32Accumulator = 3 +}; + +enum class EQuantizedComputeProfile { + UNDEFINED = 0, + GenericRecognized = 1, + SignedInt8SymmetricPerTensorRank2 = 2 +}; + enum class EQuantizedLayout { - UNDEFINED = 0, Plain = 1, Transposed = 2, PackedCPU = 3, TiledAlpaka = 4 + UNDEFINED = 0, Plain = 1, Transposed = 2, PackedCPU = 3, PlainDevice = 4, TiledAlpaka = 5 +}; + + +enum class EQuantizedShapePolicy { + UNDEFINED = 0, + Exact = 1, + ExactTooSmall = 2, + PaddedCandidate = 3, + Fallback = 4, + Unsupported = 5 +}; + +struct QuantizedMatMulShapePolicy { + EQuantizedShapePolicy policy = EQuantizedShapePolicy::UNDEFINED; + std::size_t logicalM = 0; + std::size_t logicalK = 0; + std::size_t logicalN = 0; + std::size_t physicalM = 0; + std::size_t physicalK = 0; + std::size_t physicalN = 0; + std::size_t logicalMacs = 0; + std::size_t physicalMacs = 0; + std::size_t minimumOptimizedMacs = 0; + bool belowMinimumWork = false; + double paddingWorkRatio = 1.0; + std::string reason; }; struct QuantizedTensorStorage { @@ -104,11 +156,10 @@ struct QuantizedTensorStorage { QuantizationInfo quantization; std::vector shape; + EQuantizedBackend residentBackend = EQuantizedBackend::UNDEFINED; + bool isConstant = false; - bool isPersistent = true; - bool isTransient = false; bool isDeviceResident = false; - std::size_t byteSize = 0; }; inline std::size_t QuantizedStorageElementSize(EQuantizedStorageType type) @@ -159,19 +210,43 @@ struct QuantizedLoweringPlan { EQuantizedStorageType accumulatorStorage = EQuantizedStorageType::UNDEFINED; EQuantizedStorageType outputStorage = EQuantizedStorageType::UNDEFINED; + EQuantizedCarrierMode inputCarrierMode = EQuantizedCarrierMode::UNDEFINED; + EQuantizedOutputMode outputMode = EQuantizedOutputMode::UNDEFINED; + EQuantizedComputeProfile computeProfile = EQuantizedComputeProfile::UNDEFINED; + std::string capabilityTag; + QuantizedMatMulShapePolicy shapePolicy; + std::string weightStorageTensor; EQuantizedLayout weightLayout = EQuantizedLayout::UNDEFINED; std::vector consumedOperatorIndices; bool preservesQuantizationSemantics = false; - bool hasBaselineLowering = false; - bool hasOptimizedLowering = false; bool isMetadataOnly = false; - bool usesInt32Accumulator = false; - bool usesPrequantizedWeights = false; + bool supportsPrequantizedInputCarrier = false; bool suppressesGraphOperators = false; }; +inline bool QuantizedPlanUsesInt32Accumulator(const QuantizedLoweringPlan &plan) +{ + return plan.accumulatorStorage == EQuantizedStorageType::Int32Accumulator; +} + +inline bool QuantizedPlanUsesPrequantizedWeights(const QuantizedLoweringPlan &plan) +{ + return !plan.weightStorageTensor.empty(); +} + +inline bool IsOptimizedQuantizedPlainDevicePlan(const QuantizedLoweringPlan &plan) +{ + return IsQuantizedLoweringOptimized(plan.status) && QuantizedPlanUsesPrequantizedWeights(plan) && + plan.weightLayout == EQuantizedLayout::PlainDevice; +} + +inline bool IsOptimizedQuantizedAlpakaPlainDevicePlan(const QuantizedLoweringPlan &plan) +{ + return plan.backend == EQuantizedBackend::ALPAKA && IsOptimizedQuantizedPlainDevicePlan(plan); +} + struct QuantizedGemmRegion { // Quantized carrier tensors, i.e. outputs of quantization boundaries. std::string inputTensor; diff --git a/core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx b/core/inc/SOFIE/SOFIE_Quantized.hxx similarity index 87% rename from core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx rename to core/inc/SOFIE/SOFIE_Quantized.hxx index 2d6c3da..464ed9f 100644 --- a/core/inc/SOFIE/SOFIE_QuantizedRuntime.hxx +++ b/core/inc/SOFIE/SOFIE_Quantized.hxx @@ -1,5 +1,5 @@ -#ifndef SOFIE_SOFIE_QUANTIZED_RUNTIME -#define SOFIE_SOFIE_QUANTIZED_RUNTIME +#ifndef SOFIE_QUANTIZED +#define SOFIE_QUANTIZED #include #include @@ -11,7 +11,53 @@ namespace SOFIE { -// GEMM only, need to abstract for multiple operators / backends. +// Portable quantized runtime helpers used by generated lowered quantized operators. + +inline bool NearlyEqualQuantizedScale(double lhs, double rhs) +{ + const auto scale = std::max({1.0, std::fabs(lhs), std::fabs(rhs)}); + return std::fabs(lhs - rhs) <= (1.0e-12 * scale); +} + +inline bool MakeQuantizedFixedPointMultiplier(double realMultiplier, std::int64_t &multiplier, int &shift) +{ + if (!std::isfinite(realMultiplier) || realMultiplier <= 0.0) { + return false; + } + + int exponent = 0; + const double significand = std::frexp(realMultiplier, &exponent); + auto q31 = static_cast(std::llround(significand * 2147483648.0)); + if (q31 == (std::int64_t{1} << 31)) { + q31 /= 2; + ++exponent; + } + + const int candidateShift = 31 - exponent; + if (q31 <= 0 || candidateShift < 0 || candidateShift >= 62) { + return false; + } + + multiplier = q31; + shift = candidateShift; + return true; +} + +inline bool MakeExactIntegerScaleMultiplier(double scale, std::int64_t &multiplier, int &shift) +{ + if (!std::isfinite(scale) || scale < 0.0) { + return false; + } + + const auto rounded = std::llround(scale); + if (!NearlyEqualQuantizedScale(scale, static_cast(rounded))) { + return false; + } + + multiplier = rounded; + shift = 0; + return true; +} enum class EQuantizedGemmActivation { None = 0, @@ -265,4 +311,4 @@ inline void QuantizedGemm_Call(float *output, const float *input, const WeightT } // namespace SOFIE -#endif // SOFIE_SOFIE_QUANTIZED_RUNTIME +#endif // SOFIE_QUANTIZED diff --git a/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx new file mode 100644 index 0000000..4d8a0ce --- /dev/null +++ b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx @@ -0,0 +1,918 @@ +#ifndef SOFIE_QUANTIZED_ALPAKA +#define SOFIE_QUANTIZED_ALPAKA + +#include +#include +#include +#include + +#ifdef SOFIE_USE_CUBLASLT +#include +#include +#endif + +namespace SOFIE { + +enum class EQuantizedCudaWeightType { + Int8, + UInt8 +}; + +enum class EQuantizedCudaEpilogueMode { + ExactFakeQuant, + Quantized +}; + +enum class EQuantizedCudaInputCarrier { + Float, + Int8 +}; + +enum class EQuantizedCudaOutputCarrier { + Float, + Int8, + UInt8 +}; + +struct QuantizedGemmCudaLtParams { + std::size_t m = 0; + std::size_t n = 0; + std::size_t k = 0; + double inputScale = 1.0; + double weightScale = 1.0; + double biasScale = 1.0; + double outputScale = 1.0; + std::int32_t inputZeroPoint = 0; + std::int32_t weightZeroPoint = 0; + std::int32_t biasZeroPoint = 0; + std::int32_t outputZeroPoint = 0; + std::int32_t inputQMin = -128; + std::int32_t inputQMax = 127; + std::int32_t biasQMin = -2147483648; + std::int32_t biasQMax = 2147483647; + std::int32_t outputQMin = -128; + std::int32_t outputQMax = 127; + bool hasBias = false; + bool hasRelu = false; + std::size_t maxWorkspaceBytes = 32ULL * 1024ULL * 1024ULL; + EQuantizedCudaEpilogueMode epilogueMode = EQuantizedCudaEpilogueMode::ExactFakeQuant; + EQuantizedCudaInputCarrier inputCarrier = EQuantizedCudaInputCarrier::Float; + EQuantizedCudaOutputCarrier outputCarrier = EQuantizedCudaOutputCarrier::Float; + EQuantizedCudaWeightType weightType = EQuantizedCudaWeightType::Int8; + bool enableAutotuning = true; + int autotuneIterations = 3; + double accumulatorToOutputScale = 0.0; +}; + +#ifndef SOFIE_USE_CUBLASLT + +using QuantizedGemmCudaStream = void *; + +struct QuantizedGemmCudaLtState {}; + +struct QuantizedGemmCudaLtProfile { + float inputQuantizeMs = 0.0f; + float gemmMs = 0.0f; + float epilogueMs = 0.0f; + float totalMs = 0.0f; + std::size_t workspaceSize = 0; + int heuristicResultCount = 0; + int selectedHeuristicIndex = 0; + float autotuneMs = 0.0f; + int autotunedCandidateCount = 0; + float selectedCandidateMs = 0.0f; + std::size_t accumulatorBytes = 0; + std::size_t outputBytes = 0; + std::size_t epilogueTrafficBytes = 0; + bool directOutputCarrier = false; +}; + +inline void QuantizedGemmCudaLt_SetProfiling(bool) {} +inline bool QuantizedGemmCudaLt_ProfilingEnabled() { return false; } +inline QuantizedGemmCudaLtProfile QuantizedGemmCudaLt_GetLastProfile() { return {}; } + +inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &, QuantizedGemmCudaStream, void *, const float *, + const void *, const float *, const QuantizedGemmCudaLtParams &) +{ + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM path was selected, but SOFIE_USE_CUBLASLT is not enabled"); +} + + +#else + +using QuantizedGemmCudaStream = cudaStream_t; + +namespace INTERNAL { + +inline void CheckCudaStatus(cudaError_t status, const char *where) +{ + if (status != cudaSuccess) { + throw std::runtime_error(std::string("SOFIE CUDA quantized GEMM failure in ") + where + ": " + + cudaGetErrorString(status)); + } +} + +inline void CheckCublasLtStatus(cublasStatus_t status, const char *where) +{ + if (status != CUBLAS_STATUS_SUCCESS) { + throw std::runtime_error(std::string("SOFIE cuBLASLt quantized GEMM failure in ") + where + + ": status " + std::to_string(static_cast(status))); + } +} + +__device__ inline std::int32_t QuantizedCudaClamp(std::int32_t value, std::int32_t qmin, std::int32_t qmax) +{ + return value < qmin ? qmin : (value > qmax ? qmax : value); +} + +__device__ inline std::int32_t QuantizedCudaQuantizeClamp(double value, double scale, std::int32_t zero, + std::int32_t qmin, std::int32_t qmax) +{ + const auto quantized = static_cast(nearbyint((value / scale) + static_cast(zero))); + return QuantizedCudaClamp(quantized, qmin, qmax); +} + +__global__ void QuantizedGemmCudaQuantizeInputKernel(const float *input, std::int8_t *inputQuantized, + std::size_t elements, double scale, std::int32_t zero, + std::int32_t qmin, std::int32_t qmax) +{ + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= elements) + return; + inputQuantized[idx] = static_cast(QuantizedCudaQuantizeClamp(static_cast(input[idx]), scale, + zero, qmin, qmax)); +} + +__global__ void QuantizedGemmCudaBiasOutputOffsetKernel(float *__restrict__ biasOutputOffset, + const float *__restrict__ bias, + QuantizedGemmCudaLtParams params) +{ + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= params.n) + return; + + int bq = __float2int_rn((bias[idx] / static_cast(params.biasScale)) + + static_cast(params.biasZeroPoint)); + bq = QuantizedCudaClamp(bq, params.biasQMin, params.biasQMax); + biasOutputOffset[idx] = + (static_cast(bq - params.biasZeroPoint) * static_cast(params.biasScale) / + static_cast(params.outputScale)) + + static_cast(params.outputZeroPoint); +} + +template +__global__ void QuantizedGemmCudaQuantizedEpilogueKernel(OutputT *__restrict__ output, + const std::int32_t *__restrict__ accumulator, + const float *__restrict__ biasOutputOffset, + QuantizedGemmCudaLtParams params) +{ + const std::size_t elements = params.m * params.n; + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= elements) + return; + + const std::size_t col = idx % params.n; + const float offset = HasBias ? biasOutputOffset[col] : static_cast(params.outputZeroPoint); + int yq = __float2int_rn( + __fmaf_rn(static_cast(accumulator[idx]), static_cast(params.accumulatorToOutputScale), offset)); + if constexpr (HasRelu) { + if (yq < params.outputZeroPoint) + yq = params.outputZeroPoint; + } + yq = QuantizedCudaClamp(yq, params.outputQMin, params.outputQMax); + output[idx] = static_cast(yq); +} + +__global__ void QuantizedGemmCudaEpilogueKernel(float *output, const std::int32_t *accumulator, const float *bias, + QuantizedGemmCudaLtParams params) +{ + const std::size_t elements = params.m * params.n; + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= elements) + return; + + const std::size_t col = idx % params.n; + double real = static_cast(accumulator[idx]) * params.inputScale * params.weightScale; + if (params.hasBias && bias != nullptr) { + const auto bq = QuantizedCudaQuantizeClamp(static_cast(bias[col]), params.biasScale, + params.biasZeroPoint, params.biasQMin, params.biasQMax); + real += static_cast(bq - params.biasZeroPoint) * params.biasScale; + } + + auto yq = QuantizedCudaQuantizeClamp(real, params.outputScale, params.outputZeroPoint, + params.outputQMin, params.outputQMax); + if (params.hasRelu && yq < params.outputZeroPoint) + yq = params.outputZeroPoint; + yq = QuantizedCudaClamp(yq, params.outputQMin, params.outputQMax); + output[idx] = static_cast(static_cast(yq - params.outputZeroPoint) * params.outputScale); +} + +inline void SetRowMajorLayout(cublasLtMatrixLayout_t layout) +{ + const cublasLtOrder_t order = CUBLASLT_ORDER_ROW; + CheckCublasLtStatus(cublasLtMatrixLayoutSetAttribute(layout, CUBLASLT_MATRIX_LAYOUT_ORDER, &order, sizeof(order)), + "cublasLtMatrixLayoutSetAttribute(row-major)"); +} + +inline cublasLtMatrixLayout_t CreateRowMajorLayout(cudaDataType_t type, std::uint64_t rows, std::uint64_t cols, + std::int64_t leadingDimension) +{ + cublasLtMatrixLayout_t layout = nullptr; + CheckCublasLtStatus(cublasLtMatrixLayoutCreate(&layout, type, rows, cols, leadingDimension), + "cublasLtMatrixLayoutCreate"); + SetRowMajorLayout(layout); + return layout; +} + +inline void DestroyLayout(cublasLtMatrixLayout_t &layout) +{ + if (layout != nullptr) { + cublasLtMatrixLayoutDestroy(layout); + layout = nullptr; + } +} + +} // namespace INTERNAL + +struct QuantizedGemmCudaLtProfile { + float inputQuantizeMs = 0.0f; + float gemmMs = 0.0f; + float epilogueMs = 0.0f; + float totalMs = 0.0f; + std::size_t workspaceSize = 0; + int heuristicResultCount = 0; + int selectedHeuristicIndex = 0; + float autotuneMs = 0.0f; + int autotunedCandidateCount = 0; + float selectedCandidateMs = 0.0f; + std::size_t accumulatorBytes = 0; + std::size_t outputBytes = 0; + std::size_t epilogueTrafficBytes = 0; + bool directOutputCarrier = false; +}; + +inline bool &QuantizedGemmCudaLt_ProfileEnabledStorage() +{ + static bool enabled = false; + return enabled; +} + +inline QuantizedGemmCudaLtProfile &QuantizedGemmCudaLt_LastProfileStorage() +{ + static QuantizedGemmCudaLtProfile profile{}; + return profile; +} + +inline void QuantizedGemmCudaLt_SetProfiling(bool enabled) +{ + QuantizedGemmCudaLt_ProfileEnabledStorage() = enabled; +} + +inline bool QuantizedGemmCudaLt_ProfilingEnabled() +{ + return QuantizedGemmCudaLt_ProfileEnabledStorage(); +} + +inline QuantizedGemmCudaLtProfile QuantizedGemmCudaLt_GetLastProfile() +{ + return QuantizedGemmCudaLt_LastProfileStorage(); +} + +struct QuantizedGemmCudaLtEventTimer { + cudaEvent_t totalStart = nullptr; + cudaEvent_t inputStop = nullptr; + cudaEvent_t gemmStop = nullptr; + cudaEvent_t epilogueStop = nullptr; + bool active = false; + + explicit QuantizedGemmCudaLtEventTimer(bool enabled) + : active(enabled) + { + if (!active) + return; + INTERNAL::CheckCudaStatus(cudaEventCreate(&totalStart), "cudaEventCreate(totalStart)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&inputStop), "cudaEventCreate(inputStop)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&gemmStop), "cudaEventCreate(gemmStop)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&epilogueStop), "cudaEventCreate(epilogueStop)"); + } + + QuantizedGemmCudaLtEventTimer(const QuantizedGemmCudaLtEventTimer &) = delete; + QuantizedGemmCudaLtEventTimer &operator=(const QuantizedGemmCudaLtEventTimer &) = delete; + + ~QuantizedGemmCudaLtEventTimer() + { + if (epilogueStop != nullptr) + cudaEventDestroy(epilogueStop); + if (gemmStop != nullptr) + cudaEventDestroy(gemmStop); + if (inputStop != nullptr) + cudaEventDestroy(inputStop); + if (totalStart != nullptr) + cudaEventDestroy(totalStart); + } + + void RecordStart(QuantizedGemmCudaStream stream) + { + if (active) + INTERNAL::CheckCudaStatus(cudaEventRecord(totalStart, stream), "cudaEventRecord(totalStart)"); + } + + void RecordInputStop(QuantizedGemmCudaStream stream) + { + if (active) + INTERNAL::CheckCudaStatus(cudaEventRecord(inputStop, stream), "cudaEventRecord(inputStop)"); + } + + void RecordGemmStop(QuantizedGemmCudaStream stream) + { + if (active) + INTERNAL::CheckCudaStatus(cudaEventRecord(gemmStop, stream), "cudaEventRecord(gemmStop)"); + } + + void RecordEpilogueStop(QuantizedGemmCudaStream stream, std::size_t workspaceSize, + int heuristicResultCount, int selectedHeuristicIndex, + float autotuneMs, int autotunedCandidateCount, float selectedCandidateMs, + std::size_t accumulatorBytes, std::size_t outputBytes, bool directOutputCarrier) + { + if (!active) + return; + INTERNAL::CheckCudaStatus(cudaEventRecord(epilogueStop, stream), "cudaEventRecord(epilogueStop)"); + INTERNAL::CheckCudaStatus(cudaEventSynchronize(epilogueStop), "cudaEventSynchronize(epilogueStop)"); + + QuantizedGemmCudaLtProfile profile{}; + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&profile.inputQuantizeMs, totalStart, inputStop), + "cudaEventElapsedTime(inputQuantize)"); + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&profile.gemmMs, inputStop, gemmStop), + "cudaEventElapsedTime(gemm)"); + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&profile.epilogueMs, gemmStop, epilogueStop), + "cudaEventElapsedTime(epilogue)"); + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&profile.totalMs, totalStart, epilogueStop), + "cudaEventElapsedTime(total)"); + profile.workspaceSize = workspaceSize; + profile.heuristicResultCount = heuristicResultCount; + profile.selectedHeuristicIndex = selectedHeuristicIndex; + profile.autotuneMs = autotuneMs; + profile.autotunedCandidateCount = autotunedCandidateCount; + profile.selectedCandidateMs = selectedCandidateMs; + profile.accumulatorBytes = accumulatorBytes; + profile.outputBytes = outputBytes; + profile.directOutputCarrier = directOutputCarrier; + profile.epilogueTrafficBytes = directOutputCarrier ? outputBytes : (accumulatorBytes + outputBytes); + QuantizedGemmCudaLt_LastProfileStorage() = profile; + } +}; + +struct QuantizedGemmCudaLtState { + cublasLtHandle_t fHandle = nullptr; + cublasLtMatmulDesc_t fOperation = nullptr; + cublasLtMatrixLayout_t fALayout = nullptr; + cublasLtMatrixLayout_t fBLayout = nullptr; + cublasLtMatrixLayout_t fCLayout = nullptr; + cublasLtMatmulPreference_t fPreference = nullptr; + static constexpr int kMaxHeuristicResults = 8; + cublasLtMatmulHeuristicResult_t fHeuristicResults[kMaxHeuristicResults]{}; + cublasLtMatmulHeuristicResult_t fHeuristic{}; + int fHeuristicResultCount = 0; + int fSelectedHeuristicIndex = 0; + std::size_t fWorkspaceSize = 0; + std::size_t fWorkspaceAllocatedBytes = 0; + std::size_t fWorkspaceLimitBytes = 0; + void *fWorkspace = nullptr; + bool fAutotuned = false; + float fAutotuneMs = 0.0f; + int fAutotunedCandidateCount = 0; + float fSelectedCandidateMs = 0.0f; + std::int8_t *fInputQuantized = nullptr; + std::int32_t *fAccumulator = nullptr; + float *fBiasOutputOffset = nullptr; + std::size_t fInputQuantizedBytes = 0; + std::size_t fAccumulatorBytes = 0; + std::size_t fBiasOutputOffsetBytes = 0; + const float *fBiasOutputOffsetSource = nullptr; + double fBiasOutputOffsetBiasScale = 0.0; + double fBiasOutputOffsetOutputScale = 0.0; + std::int32_t fBiasOutputOffsetBiasZeroPoint = 0; + std::int32_t fBiasOutputOffsetOutputZeroPoint = 0; + std::int32_t fBiasOutputOffsetBiasQMin = 0; + std::int32_t fBiasOutputOffsetBiasQMax = 0; + std::size_t fBiasOutputOffsetN = 0; + std::size_t fM = 0; + std::size_t fN = 0; + std::size_t fK = 0; + bool fInitialized = false; + + QuantizedGemmCudaLtState() = default; + QuantizedGemmCudaLtState(const QuantizedGemmCudaLtState &) = delete; + QuantizedGemmCudaLtState &operator=(const QuantizedGemmCudaLtState &) = delete; + + QuantizedGemmCudaLtState(QuantizedGemmCudaLtState &&other) noexcept + { + MoveFrom(other); + } + + QuantizedGemmCudaLtState &operator=(QuantizedGemmCudaLtState &&other) noexcept + { + if (this != &other) { + Reset(); + MoveFrom(other); + } + return *this; + } + + ~QuantizedGemmCudaLtState() { Reset(); } + + void Reset() noexcept + { + if (fPreference != nullptr) { + cublasLtMatmulPreferenceDestroy(fPreference); + fPreference = nullptr; + } + INTERNAL::DestroyLayout(fCLayout); + INTERNAL::DestroyLayout(fBLayout); + INTERNAL::DestroyLayout(fALayout); + if (fOperation != nullptr) { + cublasLtMatmulDescDestroy(fOperation); + fOperation = nullptr; + } + if (fHandle != nullptr) { + cublasLtDestroy(fHandle); + fHandle = nullptr; + } + if (fWorkspace != nullptr) { + cudaFree(fWorkspace); + fWorkspace = nullptr; + } + if (fInputQuantized != nullptr) { + cudaFree(fInputQuantized); + fInputQuantized = nullptr; + } + if (fAccumulator != nullptr) { + cudaFree(fAccumulator); + fAccumulator = nullptr; + } + if (fBiasOutputOffset != nullptr) { + cudaFree(fBiasOutputOffset); + fBiasOutputOffset = nullptr; + } + for (auto &heuristic : fHeuristicResults) + heuristic = cublasLtMatmulHeuristicResult_t{}; + fHeuristic = cublasLtMatmulHeuristicResult_t{}; + fHeuristicResultCount = 0; + fSelectedHeuristicIndex = 0; + fWorkspaceSize = 0; + fWorkspaceAllocatedBytes = 0; + fWorkspaceLimitBytes = 0; + fWorkspace = nullptr; + fAutotuned = false; + fAutotuneMs = 0.0f; + fAutotunedCandidateCount = 0; + fSelectedCandidateMs = 0.0f; + fInputQuantizedBytes = 0; + fAccumulatorBytes = 0; + fBiasOutputOffsetBytes = 0; + fBiasOutputOffsetSource = nullptr; + fBiasOutputOffsetBiasScale = 0.0; + fBiasOutputOffsetOutputScale = 0.0; + fBiasOutputOffsetBiasZeroPoint = 0; + fBiasOutputOffsetOutputZeroPoint = 0; + fBiasOutputOffsetBiasQMin = 0; + fBiasOutputOffsetBiasQMax = 0; + fBiasOutputOffsetN = 0; + fM = 0; + fN = 0; + fK = 0; + fInitialized = false; + } + + void Initialize(const QuantizedGemmCudaLtParams ¶ms) + { + if (fInitialized && fM == params.m && fN == params.n && fK == params.k && + fWorkspaceLimitBytes == params.maxWorkspaceBytes) + return; + + Reset(); + try { + INTERNAL::CheckCublasLtStatus(cublasLtCreate(&fHandle), "cublasLtCreate"); + INTERNAL::CheckCublasLtStatus(cublasLtMatmulDescCreate(&fOperation, CUBLAS_COMPUTE_32I, CUDA_R_32I), + "cublasLtMatmulDescCreate"); + + const cublasOperation_t transA = CUBLAS_OP_N; + const cublasOperation_t transB = CUBLAS_OP_T; + INTERNAL::CheckCublasLtStatus(cublasLtMatmulDescSetAttribute(fOperation, CUBLASLT_MATMUL_DESC_TRANSA, + &transA, sizeof(transA)), + "cublasLtMatmulDescSetAttribute(transA)"); + INTERNAL::CheckCublasLtStatus(cublasLtMatmulDescSetAttribute(fOperation, CUBLASLT_MATMUL_DESC_TRANSB, + &transB, sizeof(transB)), + "cublasLtMatmulDescSetAttribute(transB)"); + + fALayout = INTERNAL::CreateRowMajorLayout(CUDA_R_8I, params.m, params.k, + static_cast(params.k)); + fBLayout = INTERNAL::CreateRowMajorLayout(CUDA_R_8I, params.n, params.k, + static_cast(params.k)); + fCLayout = INTERNAL::CreateRowMajorLayout(CUDA_R_32I, params.m, params.n, + static_cast(params.n)); + + INTERNAL::CheckCublasLtStatus(cublasLtMatmulPreferenceCreate(&fPreference), + "cublasLtMatmulPreferenceCreate"); + fWorkspaceLimitBytes = params.maxWorkspaceBytes; + INTERNAL::CheckCublasLtStatus(cublasLtMatmulPreferenceSetAttribute( + fPreference, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, + &fWorkspaceLimitBytes, sizeof(fWorkspaceLimitBytes)), + "cublasLtMatmulPreferenceSetAttribute(workspace)"); + + INTERNAL::CheckCublasLtStatus(cublasLtMatmulAlgoGetHeuristic(fHandle, fOperation, fALayout, fBLayout, + fCLayout, fCLayout, fPreference, + kMaxHeuristicResults, fHeuristicResults, + &fHeuristicResultCount), + "cublasLtMatmulAlgoGetHeuristic"); + if (fHeuristicResultCount == 0) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM found no algorithm for the selected int8 shape"); + } + + fSelectedHeuristicIndex = 0; + fHeuristic = fHeuristicResults[fSelectedHeuristicIndex]; + fWorkspaceSize = fHeuristic.workspaceSize; + for (int i = 0; i < fHeuristicResultCount; ++i) { + if (fHeuristicResults[i].workspaceSize > fWorkspaceAllocatedBytes) + fWorkspaceAllocatedBytes = fHeuristicResults[i].workspaceSize; + } + if (fWorkspaceAllocatedBytes > 0) { + INTERNAL::CheckCudaStatus(cudaMalloc(&fWorkspace, fWorkspaceAllocatedBytes), "cudaMalloc(cuBLASLt workspace)"); + } + + fM = params.m; + fN = params.n; + fK = params.k; + fInitialized = true; + } catch (...) { + Reset(); + throw; + } + } + + void EnsureTemporaryBuffers(const QuantizedGemmCudaLtParams ¶ms) + { + const std::size_t requiredInputBytes = params.m * params.k * sizeof(std::int8_t); + if (requiredInputBytes > fInputQuantizedBytes) { + if (fInputQuantized != nullptr) { + INTERNAL::CheckCudaStatus(cudaFree(fInputQuantized), "cudaFree(inputQuantized cache)"); + fInputQuantized = nullptr; + fInputQuantizedBytes = 0; + } + INTERNAL::CheckCudaStatus(cudaMalloc(&fInputQuantized, requiredInputBytes), + "cudaMalloc(inputQuantized cache)"); + fInputQuantizedBytes = requiredInputBytes; + } + + const std::size_t requiredAccumulatorBytes = params.m * params.n * sizeof(std::int32_t); + if (requiredAccumulatorBytes > fAccumulatorBytes) { + if (fAccumulator != nullptr) { + INTERNAL::CheckCudaStatus(cudaFree(fAccumulator), "cudaFree(accumulator cache)"); + fAccumulator = nullptr; + fAccumulatorBytes = 0; + } + INTERNAL::CheckCudaStatus(cudaMalloc(&fAccumulator, requiredAccumulatorBytes), + "cudaMalloc(accumulator cache)"); + fAccumulatorBytes = requiredAccumulatorBytes; + } + } + + const float *EnsureBiasOutputOffsetBuffer(const QuantizedGemmCudaLtParams ¶ms, const float *bias, + QuantizedGemmCudaStream stream) + { + if (bias == nullptr || !params.hasBias) + return nullptr; + + const std::size_t requiredBytes = params.n * sizeof(float); + if (requiredBytes > fBiasOutputOffsetBytes) { + if (fBiasOutputOffset != nullptr) { + INTERNAL::CheckCudaStatus(cudaFree(fBiasOutputOffset), "cudaFree(bias output offset cache)"); + fBiasOutputOffset = nullptr; + fBiasOutputOffsetBytes = 0; + } + INTERNAL::CheckCudaStatus(cudaMalloc(&fBiasOutputOffset, requiredBytes), + "cudaMalloc(bias output offset cache)"); + fBiasOutputOffsetBytes = requiredBytes; + fBiasOutputOffsetSource = nullptr; + } + + const bool cacheValid = fBiasOutputOffsetSource == bias && fBiasOutputOffsetN == params.n && + fBiasOutputOffsetBiasScale == params.biasScale && + fBiasOutputOffsetOutputScale == params.outputScale && + fBiasOutputOffsetBiasZeroPoint == params.biasZeroPoint && + fBiasOutputOffsetOutputZeroPoint == params.outputZeroPoint && + fBiasOutputOffsetBiasQMin == params.biasQMin && + fBiasOutputOffsetBiasQMax == params.biasQMax; + if (!cacheValid) { + constexpr int threads = 256; + const int blocks = static_cast((params.n + threads - 1) / threads); + INTERNAL::QuantizedGemmCudaBiasOutputOffsetKernel<<>>( + fBiasOutputOffset, bias, params); + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaBiasOutputOffsetKernel launch"); + fBiasOutputOffsetSource = bias; + fBiasOutputOffsetN = params.n; + fBiasOutputOffsetBiasScale = params.biasScale; + fBiasOutputOffsetOutputScale = params.outputScale; + fBiasOutputOffsetBiasZeroPoint = params.biasZeroPoint; + fBiasOutputOffsetOutputZeroPoint = params.outputZeroPoint; + fBiasOutputOffsetBiasQMin = params.biasQMin; + fBiasOutputOffsetBiasQMax = params.biasQMax; + } + + return fBiasOutputOffset; + } + + std::int8_t *InputQuantizedBuffer() const { return fInputQuantized; } + std::int32_t *AccumulatorBuffer() const { return fAccumulator; } + std::size_t AccumulatorBytes() const { return fAccumulatorBytes; } + std::size_t WorkspaceSize() const { return fWorkspaceSize; } + int HeuristicResultCount() const { return fHeuristicResultCount; } + int SelectedHeuristicIndex() const { return fSelectedHeuristicIndex; } + float AutotuneMs() const { return fAutotuneMs; } + int AutotunedCandidateCount() const { return fAutotunedCandidateCount; } + float SelectedCandidateMs() const { return fSelectedCandidateMs; } + + void Autotune(std::int32_t *accumulator, const std::int8_t *inputQuantized, const std::int8_t *weightQuantized, + const QuantizedGemmCudaLtParams ¶ms, QuantizedGemmCudaStream stream) + { + if (fAutotuned || !params.enableAutotuning || fHeuristicResultCount <= 1) { + fAutotuned = true; + return; + } + + cudaEvent_t totalStart = nullptr; + cudaEvent_t totalStop = nullptr; + cudaEvent_t candidateStart = nullptr; + cudaEvent_t candidateStop = nullptr; + INTERNAL::CheckCudaStatus(cudaEventCreate(&totalStart), "cudaEventCreate(autotuneTotalStart)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&totalStop), "cudaEventCreate(autotuneTotalStop)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&candidateStart), "cudaEventCreate(autotuneCandidateStart)"); + INTERNAL::CheckCudaStatus(cudaEventCreate(&candidateStop), "cudaEventCreate(autotuneCandidateStop)"); + + const std::int32_t alpha = 1; + const std::int32_t beta = 0; + const int iterations = params.autotuneIterations > 0 ? params.autotuneIterations : 1; + float bestMs = 0.0f; + int bestIndex = fSelectedHeuristicIndex; + int measuredCandidates = 0; + INTERNAL::CheckCudaStatus(cudaEventRecord(totalStart, stream), "cudaEventRecord(autotuneTotalStart)"); + for (int i = 0; i < fHeuristicResultCount; ++i) { + const auto warmupStatus = cublasLtMatmul(fHandle, fOperation, &alpha, inputQuantized, fALayout, + weightQuantized, fBLayout, &beta, accumulator, fCLayout, + accumulator, fCLayout, &fHeuristicResults[i].algo, fWorkspace, + fWorkspaceAllocatedBytes, stream); + if (warmupStatus != CUBLAS_STATUS_SUCCESS) + continue; + INTERNAL::CheckCudaStatus(cudaEventRecord(candidateStart, stream), "cudaEventRecord(autotuneCandidateStart)"); + bool candidateOk = true; + for (int iteration = 0; iteration < iterations; ++iteration) { + const auto status = cublasLtMatmul(fHandle, fOperation, &alpha, inputQuantized, fALayout, + weightQuantized, fBLayout, &beta, accumulator, fCLayout, + accumulator, fCLayout, &fHeuristicResults[i].algo, fWorkspace, + fWorkspaceAllocatedBytes, stream); + if (status != CUBLAS_STATUS_SUCCESS) { + candidateOk = false; + break; + } + } + if (!candidateOk) + continue; + INTERNAL::CheckCudaStatus(cudaEventRecord(candidateStop, stream), "cudaEventRecord(autotuneCandidateStop)"); + INTERNAL::CheckCudaStatus(cudaEventSynchronize(candidateStop), "cudaEventSynchronize(autotuneCandidateStop)"); + float candidateMs = 0.0f; + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&candidateMs, candidateStart, candidateStop), + "cudaEventElapsedTime(autotuneCandidate)"); + candidateMs /= static_cast(iterations); + ++measuredCandidates; + if (measuredCandidates == 1 || candidateMs < bestMs) { + bestMs = candidateMs; + bestIndex = i; + } + } + INTERNAL::CheckCudaStatus(cudaEventRecord(totalStop, stream), "cudaEventRecord(autotuneTotalStop)"); + INTERNAL::CheckCudaStatus(cudaEventSynchronize(totalStop), "cudaEventSynchronize(autotuneTotalStop)"); + INTERNAL::CheckCudaStatus(cudaEventElapsedTime(&fAutotuneMs, totalStart, totalStop), + "cudaEventElapsedTime(autotuneTotal)"); + + cudaEventDestroy(candidateStop); + cudaEventDestroy(candidateStart); + cudaEventDestroy(totalStop); + cudaEventDestroy(totalStart); + + if (measuredCandidates > 0) { + fSelectedHeuristicIndex = bestIndex; + fHeuristic = fHeuristicResults[fSelectedHeuristicIndex]; + fWorkspaceSize = fHeuristic.workspaceSize; + fSelectedCandidateMs = bestMs; + fAutotunedCandidateCount = measuredCandidates; + } + fAutotuned = true; + } + + void Execute(std::int32_t *accumulator, const std::int8_t *inputQuantized, const std::int8_t *weightQuantized, + const QuantizedGemmCudaLtParams ¶ms, QuantizedGemmCudaStream stream) + { + Initialize(params); + Autotune(accumulator, inputQuantized, weightQuantized, params, stream); + const std::int32_t alpha = 1; + const std::int32_t beta = 0; + INTERNAL::CheckCublasLtStatus(cublasLtMatmul(fHandle, fOperation, &alpha, inputQuantized, fALayout, + weightQuantized, fBLayout, &beta, accumulator, fCLayout, + accumulator, fCLayout, &fHeuristic.algo, fWorkspace, + fWorkspaceAllocatedBytes, stream), + "cublasLtMatmul"); + } + +private: + void MoveFrom(QuantizedGemmCudaLtState &other) noexcept + { + fHandle = other.fHandle; + fOperation = other.fOperation; + fALayout = other.fALayout; + fBLayout = other.fBLayout; + fCLayout = other.fCLayout; + fPreference = other.fPreference; + for (int i = 0; i < kMaxHeuristicResults; ++i) + fHeuristicResults[i] = other.fHeuristicResults[i]; + fHeuristic = other.fHeuristic; + fHeuristicResultCount = other.fHeuristicResultCount; + fSelectedHeuristicIndex = other.fSelectedHeuristicIndex; + fWorkspaceSize = other.fWorkspaceSize; + fWorkspaceAllocatedBytes = other.fWorkspaceAllocatedBytes; + fWorkspaceLimitBytes = other.fWorkspaceLimitBytes; + fWorkspace = other.fWorkspace; + fAutotuned = other.fAutotuned; + fAutotuneMs = other.fAutotuneMs; + fAutotunedCandidateCount = other.fAutotunedCandidateCount; + fSelectedCandidateMs = other.fSelectedCandidateMs; + fInputQuantized = other.fInputQuantized; + fAccumulator = other.fAccumulator; + fBiasOutputOffset = other.fBiasOutputOffset; + fInputQuantizedBytes = other.fInputQuantizedBytes; + fAccumulatorBytes = other.fAccumulatorBytes; + fBiasOutputOffsetBytes = other.fBiasOutputOffsetBytes; + fBiasOutputOffsetSource = other.fBiasOutputOffsetSource; + fBiasOutputOffsetBiasScale = other.fBiasOutputOffsetBiasScale; + fBiasOutputOffsetOutputScale = other.fBiasOutputOffsetOutputScale; + fBiasOutputOffsetBiasZeroPoint = other.fBiasOutputOffsetBiasZeroPoint; + fBiasOutputOffsetOutputZeroPoint = other.fBiasOutputOffsetOutputZeroPoint; + fBiasOutputOffsetBiasQMin = other.fBiasOutputOffsetBiasQMin; + fBiasOutputOffsetBiasQMax = other.fBiasOutputOffsetBiasQMax; + fBiasOutputOffsetN = other.fBiasOutputOffsetN; + fM = other.fM; + fN = other.fN; + fK = other.fK; + fInitialized = other.fInitialized; + + other.fHandle = nullptr; + other.fOperation = nullptr; + other.fALayout = nullptr; + other.fBLayout = nullptr; + other.fCLayout = nullptr; + other.fPreference = nullptr; + for (auto &heuristic : other.fHeuristicResults) + heuristic = cublasLtMatmulHeuristicResult_t{}; + other.fHeuristic = cublasLtMatmulHeuristicResult_t{}; + other.fHeuristicResultCount = 0; + other.fSelectedHeuristicIndex = 0; + other.fWorkspaceSize = 0; + other.fWorkspaceAllocatedBytes = 0; + other.fWorkspaceLimitBytes = 0; + other.fWorkspace = nullptr; + other.fAutotuned = false; + other.fAutotuneMs = 0.0f; + other.fAutotunedCandidateCount = 0; + other.fSelectedCandidateMs = 0.0f; + other.fInputQuantized = nullptr; + other.fAccumulator = nullptr; + other.fBiasOutputOffset = nullptr; + other.fInputQuantizedBytes = 0; + other.fAccumulatorBytes = 0; + other.fBiasOutputOffsetBytes = 0; + other.fBiasOutputOffsetSource = nullptr; + other.fBiasOutputOffsetBiasScale = 0.0; + other.fBiasOutputOffsetOutputScale = 0.0; + other.fBiasOutputOffsetBiasZeroPoint = 0; + other.fBiasOutputOffsetOutputZeroPoint = 0; + other.fBiasOutputOffsetBiasQMin = 0; + other.fBiasOutputOffsetBiasQMax = 0; + other.fBiasOutputOffsetN = 0; + other.fM = 0; + other.fN = 0; + other.fK = 0; + other.fInitialized = false; + } +}; + +inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedGemmCudaStream stream, + void *output, const float *input, const void *weight, const float *bias, + const QuantizedGemmCudaLtParams ¶ms) +{ + if (output == nullptr || input == nullptr || weight == nullptr) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM received a null required pointer"); + } + if (params.m == 0 || params.n == 0 || params.k == 0) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM requires nonzero M, N, and K"); + } + if (params.weightType != EQuantizedCudaWeightType::Int8) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM currently supports signed int8 weights only"); + } + if (params.inputZeroPoint != 0 || params.weightZeroPoint != 0) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM currently requires input and weight zero points to be 0"); + } + if (params.epilogueMode == EQuantizedCudaEpilogueMode::Quantized && + params.outputCarrier == EQuantizedCudaOutputCarrier::Float) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM quantized epilogue requires an integer output carrier"); + } + if (params.epilogueMode != EQuantizedCudaEpilogueMode::Quantized && + params.outputCarrier != EQuantizedCudaOutputCarrier::Float) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM float epilogues require a float output carrier"); + } + + QuantizedGemmCudaLtParams effectiveParams = params; + if (effectiveParams.accumulatorToOutputScale == 0.0) + effectiveParams.accumulatorToOutputScale = + (effectiveParams.inputScale * effectiveParams.weightScale) / effectiveParams.outputScale; + + const std::size_t inputElements = effectiveParams.m * effectiveParams.k; + const std::size_t outputElements = effectiveParams.m * effectiveParams.n; + state.Initialize(effectiveParams); + state.EnsureTemporaryBuffers(effectiveParams); + QuantizedGemmCudaLtEventTimer profileTimer(QuantizedGemmCudaLt_ProfilingEnabled()); + profileTimer.RecordStart(stream); + + constexpr int threads = 256; + const std::int8_t *inputQuantized = nullptr; + if (effectiveParams.inputCarrier == EQuantizedCudaInputCarrier::Int8) { + inputQuantized = reinterpret_cast(input); + } else { + const int inputBlocks = static_cast((inputElements + threads - 1) / threads); + INTERNAL::QuantizedGemmCudaQuantizeInputKernel<<>>( + input, state.InputQuantizedBuffer(), inputElements, effectiveParams.inputScale, effectiveParams.inputZeroPoint, + effectiveParams.inputQMin, effectiveParams.inputQMax); + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaQuantizeInputKernel launch"); + inputQuantized = state.InputQuantizedBuffer(); + } + profileTimer.RecordInputStop(stream); + + state.Execute(state.AccumulatorBuffer(), inputQuantized, static_cast(weight), effectiveParams, + stream); + profileTimer.RecordGemmStop(stream); + + const int outputBlocks = static_cast((outputElements + threads - 1) / threads); + std::size_t outputBytes = outputElements * sizeof(float); + bool directOutputCarrier = false; + if (effectiveParams.epilogueMode == EQuantizedCudaEpilogueMode::Quantized) { + directOutputCarrier = true; + const float *biasOutputOffset = state.EnsureBiasOutputOffsetBuffer(effectiveParams, bias, stream); + + outputBytes = outputElements * (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8 ? sizeof(std::uint8_t) : sizeof(std::int8_t)); + if (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8) { + auto *quantizedOutput = static_cast(output); + if (effectiveParams.hasBias && bias != nullptr) { + if (effectiveParams.hasRelu) { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + } else { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + } + } else { + if (effectiveParams.hasRelu) { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + } else { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + } + } + } else { + auto *quantizedOutput = static_cast(output); + if (effectiveParams.hasBias && bias != nullptr) { + if (effectiveParams.hasRelu) { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + } else { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + } + } else { + if (effectiveParams.hasRelu) { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + } else { + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + } + } + } + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaQuantizedEpilogueKernel launch"); + } else { + auto *floatOutput = static_cast(output); + INTERNAL::QuantizedGemmCudaEpilogueKernel<<>>(floatOutput, state.AccumulatorBuffer(), bias, effectiveParams); + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaEpilogueKernel launch"); + } + profileTimer.RecordEpilogueStop(stream, state.WorkspaceSize(), state.HeuristicResultCount(), + state.SelectedHeuristicIndex(), state.AutotuneMs(), + state.AutotunedCandidateCount(), state.SelectedCandidateMs(), + state.AccumulatorBytes(), outputBytes, directOutputCarrier); +} + + +#endif // SOFIE_USE_CUBLASLT + +} // namespace SOFIE + +#endif // SOFIE_QUANTIZED_ALPAKA diff --git a/core/src/RModel.cxx b/core/src/RModel.cxx index 4b4a3f0..4a070e7 100644 --- a/core/src/RModel.cxx +++ b/core/src/RModel.cxx @@ -58,7 +58,7 @@ void RModel::BuildLoweredOperatorView(EQuantizedBackend backend) { fLoweredOperators.clear(); fLoweredConsumedOperatorIndices.clear(); - AddLoweredQuantizedGemmOperators(backend); + AddLoweredQuantizedOperators(backend); } std::vector RModel::GetTensorShape(const std::string & name) const { @@ -1540,7 +1540,7 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, lo Initialize(batchSize, verbose); BuildLoweredOperatorView(EQuantizedBackend::CPU); - AddQuantizedGeneratedHeaders(); + AddQuantizedGeneratedHeaders(EQuantizedBackend::CPU); // if having dynamic tensor we need to have a Session if (!fDynamicTensorInfos.empty()) { diff --git a/core/src/RModel_ALPAKA.cxx b/core/src/RModel_ALPAKA.cxx index 9e0e84c..e1ce168 100644 --- a/core/src/RModel_ALPAKA.cxx +++ b/core/src/RModel_ALPAKA.cxx @@ -155,6 +155,8 @@ void RModel::GenerateInitializedTensorInfo_GPU_ALPAKA() { fGC += GenerateConstantTensorCode(i); else if (i.second.type() == ETensorType::INT32) fGC += GenerateConstantTensorCode(i); + else if (i.second.type() == ETensorType::INT8) + fGC += GenerateConstantTensorCode(i); else if (i.second.type() == ETensorType::BOOL || i.second.type() == ETensorType::UINT8) @@ -170,6 +172,10 @@ void RModel::GenerateInitializedTensorInfo_GPU_ALPAKA() { fGC += "BufI321D deviceBuf_" + i.first + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + std::to_string(length) + "}));\n"; + } else if (i.second.type() == ETensorType::INT8) { + fGC += "BufI81D deviceBuf_" + i.first + + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + + std::to_string(length) + "}));\n"; } else if (i.second.type() == ETensorType::INT64) { fGC += "BufI641D deviceBuf_" + i.first + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + @@ -197,6 +203,8 @@ void RModel::GenerateTemporaryInitializedTensorContainers_GPU_ALPAKA() fGC += "std::vector tensor_" + i.first + "(" + std::to_string(length) + ");\n"; } else if (i.second.type() == ETensorType::INT32) { fGC += "std::vector tensor_" + i.first + "(" + std::to_string(length) + ");\n"; + } else if (i.second.type() == ETensorType::INT8) { + fGC += "std::vector tensor_" + i.first + "(" + std::to_string(length) + ");\n"; } else if (i.second.type() == ETensorType::INT64) { fGC += "std::vector tensor_" + i.first + "(" + std::to_string(length) + ");\n"; } else if (i.second.type() == ETensorType::BOOL || @@ -229,11 +237,15 @@ void RModel::GenerateGPU_ALPAKA_Buffers() { tensor_declaration_block += "BufI321D deviceBuf_" + i.first + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + std::to_string(length) + "}));\n"; + } else if (i.second.type == ETensorType::INT8) { + tensor_declaration_block += "BufI81D deviceBuf_" + i.first + + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + + std::to_string(length) + "}));\n"; } else if (i.second.type == ETensorType::INT64) { tensor_declaration_block += "BufI641D deviceBuf_" + i.first + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + std::to_string(length) + "}));\n"; - } else if (i.second.type == ETensorType::BOOL) { + } else if (i.second.type == ETensorType::BOOL || i.second.type == ETensorType::UINT8) { tensor_declaration_block += "BufUI81D deviceBuf_" + i.first + " = alpaka::allocBuf(devAcc, Ext1D::all(Idx{" + std::to_string(length) + "}));\n"; @@ -326,7 +338,8 @@ std::string RModel::GenerateImplSignature_GPU_ALPAKA(bool isdecl) { if (type == ETensorType::DOUBLE) return "ViewConstD1D"; if (type == ETensorType::INT32) return "ViewConstI321D"; if (type == ETensorType::INT64) return "ViewConstI641D"; - if (type == ETensorType::BOOL) return "ViewConstUI81D"; + if (type == ETensorType::INT8) return "ViewConstI81D"; + if (type == ETensorType::BOOL || type == ETensorType::UINT8) return "ViewConstUI81D"; throw std::runtime_error("sofie: input tensor " + name + " is of a data type which is not yet supported."); }; @@ -378,7 +391,8 @@ void RModel::GenerateOutput_GPU_ALPAKA() { if (type == ETensorType::DOUBLE) return "ViewConstD1D"; if (type == ETensorType::INT32) return "ViewConstI321D"; if (type == ETensorType::INT64) return "ViewConstI641D"; - if (type == ETensorType::BOOL) return "ViewConstUI81D"; + if (type == ETensorType::INT8) return "ViewConstI81D"; + if (type == ETensorType::BOOL || type == ETensorType::UINT8) return "ViewConstUI81D"; throw std::runtime_error("sofie: input tensor " + name + " is of an unsupported data type."); }; @@ -416,7 +430,9 @@ void RModel::GenerateOutput_GPU_ALPAKA() { if (fVerbose) std::cout << "Generating code for operator .... " << op_idx << std::endl; - if (fSkipOperators.count(op_idx)) continue; + if (fSkipOperators.count(op_idx) || fLoweredConsumedOperatorIndices.count(op_idx) != 0) continue; + auto loweredIt = fLoweredOperators.find(op_idx); + ROperator *op = (loweredIt != fLoweredOperators.end()) ? loweredIt->second.get() : fOperators[op_idx].get(); auto gIt = fOpToFusionGroupIdx.find(op_idx); size_t gIdx = (gIt != fOpToFusionGroupIdx.end()) ? gIt->second : SIZE_MAX; @@ -457,9 +473,9 @@ void RModel::GenerateOutput_GPU_ALPAKA() { // Chain followers: skip — their logic is inside the fused kernel } else { if (fProfile) { - fGC += RModelProfilerGPU::GenerateOperatorCode(*fOperators[op_idx], op_idx); + fGC += RModelProfilerGPU::GenerateOperatorCode(*op, op_idx); } else { - fGC += fOperators[op_idx]->Generate_GPU_ALPAKA(std::to_string(op_idx)); + fGC += op->Generate_GPU_ALPAKA(std::to_string(op_idx)); } } } @@ -477,31 +493,34 @@ void RModel::GenerateOutput_GPU_ALPAKA() { for (auto &p : dynParamNames) spanDynDecl += ", size_t " + p; - fGC += "void infer(std::span inputs, std::span outputs" + spanDynDecl + "){\n"; - - { - fGC += SP + "_infer_impl("; - bool first = true; - for (auto &p : dynParamNames) { - if (!first) fGC += ", "; - fGC += p; - first = false; - } - for (size_t i = 0; i < fInputTensorNames.size(); i++) { - if (!first) fGC += ", "; - fGC += "inputs[" + std::to_string(i) + "]"; - first = false; + const bool allFloatOutputs = sameOutputTypes && eFirstOutputType == ETensorType::FLOAT; + if (allFloatOutputs) { + fGC += "void infer(std::span inputs, std::span outputs" + spanDynDecl + "){\n"; + + { + fGC += SP + "_infer_impl("; + bool first = true; + for (auto &p : dynParamNames) { + if (!first) fGC += ", "; + fGC += p; + first = false; + } + for (size_t i = 0; i < fInputTensorNames.size(); i++) { + if (!first) fGC += ", "; + fGC += "inputs[" + std::to_string(i) + "]"; + first = false; + } + fGC += ");\n"; } - fGC += ");\n"; - } - // Copy member output buffers into caller-provided output views - for (size_t i = 0; i < outputSize; i++) { - std::string tensorName = *(fOutputTensorNames.begin() + i); - fGC += SP + "alpaka::memcpy(queue, outputs[" + std::to_string(i) + "], deviceBuf_" + tensorName + ");\n"; + // Copy member output buffers into caller-provided output views + for (size_t i = 0; i < outputSize; i++) { + std::string tensorName = *(fOutputTensorNames.begin() + i); + fGC += SP + "alpaka::memcpy(queue, outputs[" + std::to_string(i) + "], deviceBuf_" + tensorName + ");\n"; + } + fGC += SP + "alpaka::wait(queue);\n"; + fGC += "}\n\n"; } - fGC += SP + "alpaka::wait(queue);\n"; - fGC += "}\n\n"; std::string returnType; @@ -583,7 +602,10 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fGC += "\n//--- ALPAKA Kernels\n"; for (size_t id = 0; id < fOperators.size(); id++) { - if(fOperators[id]->GetKind() == OperatorKind::GEMM || fOperators[id]->GetKind() == OperatorKind::CONV) { + if (fLoweredConsumedOperatorIndices.count(id) != 0) continue; + auto loweredIt = fLoweredOperators.find(id); + ROperator *op = (loweredIt != fLoweredOperators.end()) ? loweredIt->second.get() : fOperators[id].get(); + if(op->GetKind() == OperatorKind::GEMM || op->GetKind() == OperatorKind::CONV) { OpNeedsBlas = true; } @@ -614,17 +636,17 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { // Chain followers: skip (their logic is inside the fused kernel) } else { // Unfused op: generate individual kernel struct (with dedup for single_initialized_operators) - if (single_initialized_operators.find(fOperators[id]->GetKind()) != single_initialized_operators.end()) { - if (registered_operators.find(fOperators[id]->GetKind()) == registered_operators.end()) { + if (single_initialized_operators.find(op->GetKind()) != single_initialized_operators.end()) { + if (registered_operators.find(op->GetKind()) == registered_operators.end()) { if (fVerbose) - std::cout << "Generating ALPAKA kernel for operator " << toString(fOperators[id]->GetKind()) << std::endl; - fGC += fOperators[id]->Generate_GPU_Kernel_ALPAKA(std::to_string(id)); - registered_operators.insert(fOperators[id]->GetKind()); + std::cout << "Generating ALPAKA kernel for operator " << toString(op->GetKind()) << std::endl; + fGC += op->Generate_GPU_Kernel_ALPAKA(std::to_string(id)); + registered_operators.insert(op->GetKind()); } } else { if (fVerbose) - std::cout << "Generating ALPAKA kernel for operator " << toString(fOperators[id]->GetKind()) << std::endl; - fGC += fOperators[id]->Generate_GPU_Kernel_ALPAKA(std::to_string(id)); + std::cout << "Generating ALPAKA kernel for operator " << toString(op->GetKind()) << std::endl; + fGC += op->Generate_GPU_Kernel_ALPAKA(std::to_string(id)); } } } @@ -659,6 +681,7 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fGC += "using QueueAcc = alpaka::Queue;\n\n"; fGC += "using BufF1D = alpaka::Buf;\n"; fGC += "using BufD1D = alpaka::Buf;\n"; + fGC += "using BufI81D = alpaka::Buf;\n"; fGC += "using BufI321D = alpaka::Buf;\n"; fGC += "using BufI641D = alpaka::Buf;\n"; fGC += "using BufUI81D = alpaka::Buf;\n\n"; @@ -667,6 +690,8 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fGC += "using ViewConstF1D = alpaka::ViewPlainPtr;\n"; fGC += "using ViewD1D = alpaka::ViewPlainPtr;\n"; fGC += "using ViewConstD1D = alpaka::ViewPlainPtr;\n"; + fGC += "using ViewI81D = alpaka::ViewPlainPtr;\n"; + fGC += "using ViewConstI81D = alpaka::ViewPlainPtr;\n"; fGC += "using ViewI321D = alpaka::ViewPlainPtr;\n"; fGC += "using ViewConstI321D = alpaka::ViewPlainPtr;\n"; fGC += "using ViewI641D = alpaka::ViewPlainPtr;\n"; @@ -732,12 +757,14 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { GenerateDynamicTensorInfo_GPU_ALPAKA(); for (size_t id = 0; id < fOperators.size(); id++) { - if (fSkipOperators.count(id)) continue; - fGC += fOperators[id]->GenerateInitCode_GPU_ALPAKA(); - if (fOperators[id]->GetKind() == OperatorKind::GEMM || fOperators[id]->GetKind() == OperatorKind::CONV) { + if (fSkipOperators.count(id) || fLoweredConsumedOperatorIndices.count(id) != 0) continue; + auto loweredIt = fLoweredOperators.find(id); + ROperator *op = (loweredIt != fLoweredOperators.end()) ? loweredIt->second.get() : fOperators[id].get(); + fGC += op->GenerateInitCode_GPU_ALPAKA(); + if (op->GetKind() == OperatorKind::GEMM || op->GetKind() == OperatorKind::CONV) { // GetBlasConfig() returns "" for ops that use gemmStridedBatched - // (legacy cuBLAS path, no cuBLASLt layout registration needed). - auto blasCfg = fOperators[id]->GetBlasConfig(); + // (legacy cuBLAS path, no cuBLASLt layout needed). + auto blasCfg = op->GetBlasConfig(); if (!blasCfg.empty()) fGC += "\nblas.addLayoutConfig("+blasCfg+");\n"; } @@ -751,6 +778,9 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fusedGroupsEmitted.clear(); for (size_t id = 0; id < fOperators.size(); id++) { + if (fLoweredConsumedOperatorIndices.count(id) != 0) continue; + auto loweredIt = fLoweredOperators.find(id); + ROperator *op = (loweredIt != fLoweredOperators.end()) ? loweredIt->second.get() : fOperators[id].get(); // Same as the kernel-struct loop above: fused activation ops must still // declare their member variable (e.g. `leakyReluKernel`) even though // their Generate_GPU_ALPAKA call is skipped in the infer-body loop. @@ -766,17 +796,17 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fusedGroupsEmitted.insert(gIdx); } } else { - if (single_initialized_operators.find(fOperators[id]->GetKind()) != single_initialized_operators.end()) { - if (registered_operators.find(fOperators[id]->GetKind()) == registered_operators.end()) { + if (single_initialized_operators.find(op->GetKind()) != single_initialized_operators.end()) { + if (registered_operators.find(op->GetKind()) == registered_operators.end()) { if (fVerbose) - std::cout << "Declaring ALPAKA kernel for operator " << toString(fOperators[id]->GetKind()) << std::endl; - fGC += fOperators[id]->Generate_GPU_Kernel_Definitions_ALPAKA(std::to_string(id)); - registered_operators.insert(fOperators[id]->GetKind()); + std::cout << "Declaring ALPAKA kernel for operator " << toString(op->GetKind()) << std::endl; + fGC += op->Generate_GPU_Kernel_Definitions_ALPAKA(std::to_string(id)); + registered_operators.insert(op->GetKind()); } } else { if (fVerbose) - std::cout << "Declaring ALPAKA kernel for operator " << toString(fOperators[id]->GetKind()) << std::endl; - fGC += fOperators[id]->Generate_GPU_Kernel_Definitions_ALPAKA(std::to_string(id)); + std::cout << "Declaring ALPAKA kernel for operator " << toString(op->GetKind()) << std::endl; + fGC += op->Generate_GPU_Kernel_Definitions_ALPAKA(std::to_string(id)); } } } @@ -825,6 +855,8 @@ void RModel::GenerateGPU_ALPAKA(std::underlying_type_t options, int bat throw std::runtime_error("SOFIE GPU does not yet supports GNN Inference."); Initialize(batchSize, verbose); + BuildLoweredOperatorView(EQuantizedBackend::ALPAKA); + AddQuantizedGeneratedHeaders(EQuantizedBackend::ALPAKA); FuseGemmActivations_GPU(); // must run before elementwise fusion (redirects tensors) ComputeEltwiseFusionGroups(); @@ -848,6 +880,15 @@ void RModel::GenerateGPU_ALPAKA(std::underlying_type_t options, int bat void RModel::MoveInitializedTensorsToBuffers_ALPAKA(){ for (auto &i : fInitializedTensors) { if (i.second.IsNotWritable()) continue; + if (HasQuantizedTensorStorage(i.first)) { + const auto &storage = GetQuantizedTensorStorage(i.first); + if (storage.layout == EQuantizedLayout::PackedCPU || !storage.isDeviceResident) + continue; + } + const auto type = i.second.type(); + if (type != ETensorType::FLOAT && type != ETensorType::INT32 && type != ETensorType::INT64 && + type != ETensorType::BOOL && type != ETensorType::INT8 && type != ETensorType::UINT8) + continue; std::string tensor_name = "tensor_" + i.first; auto length = ConvertShapeToLength(i.second.shape()); std::string slength = std::to_string(length); diff --git a/core/src/RModel_Quantization.cxx b/core/src/RModel_Quantization.cxx index f54dda4..c11f8ce 100644 --- a/core/src/RModel_Quantization.cxx +++ b/core/src/RModel_Quantization.cxx @@ -40,6 +40,24 @@ EQuantizedStorageType StorageTypeForQuantizedTensor(const QuantizationInfo &info return info.isSigned ? EQuantizedStorageType::Int8 : EQuantizedStorageType::UInt8; } +std::vector QuantizeTensorToInt8(const float *data, std::size_t length, const QuantizationInfo &info) +{ + std::vector quantized(length); + for (std::size_t i = 0; i < length; ++i) { + quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); + } + return quantized; +} + +std::vector QuantizeTensorToUInt8(const float *data, std::size_t length, const QuantizationInfo &info) +{ + std::vector quantized(length); + for (std::size_t i = 0; i < length; ++i) { + quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); + } + return quantized; +} + std::string GetQuantBoundarySourceTensor(const ROperator &op) { auto inputs = op.GetOpInputTensors(); @@ -52,10 +70,6 @@ std::string GetQuantBoundarySourceTensor(const ROperator &op) QuantizedGemmCodegenContext MakeQuantizedGemmCodegenContext(const ROperator_Gemm &gemm) { QuantizedGemmCodegenContext context; - context.inputTensor = gemm.GetInputTensorName(); - context.weightTensor = gemm.GetWeightTensorName(); - context.biasTensor = gemm.GetBiasTensorName(); - context.outputTensor = gemm.GetOutputTensorName(); context.inputShape = gemm.GetInputShape(); context.weightShape = gemm.GetWeightShape(); context.outputShape = gemm.GetOutputShape(); @@ -64,36 +78,75 @@ QuantizedGemmCodegenContext MakeQuantizedGemmCodegenContext(const ROperator_Gemm context.transA = gemm.GetTransA(); context.transB = gemm.GetTransB(); context.activation = gemm.GetActivationType(); - context.indent = " "; return context; } +std::vector QuantizedGemmConsumedOperatorIndices(const QuantizedGemmRegion ®ion) +{ + std::vector indices = { region.inputQuantOpIndex, region.weightQuantOpIndex, + region.gemmOpIndex, region.outputQuantOpIndex }; + if (region.biasQuantOpIndex) { + indices.push_back(*region.biasQuantOpIndex); + } + std::sort(indices.begin(), indices.end()); + return indices; +} + +QuantizedLoweringPlan MakeAvailableQuantizedGemmPlan(const QuantizedGemmRegion ®ion, + EQuantizedBackend backend, + EQuantizedLoweringStatus status, + std::string reason, + std::string capabilityTag) +{ + QuantizedLoweringPlan plan; + plan.backend = backend; + plan.status = status; + plan.reason = std::move(reason); + plan.capabilityTag = std::move(capabilityTag); + plan.consumedOperatorIndices = QuantizedGemmConsumedOperatorIndices(region); + plan.preservesQuantizationSemantics = true; + plan.isMetadataOnly = false; + plan.suppressesGraphOperators = true; + return plan; +} + +std::vector QuantizedGemmOperatorIndices(const QuantizationModelState &state) +{ + std::vector indices; + indices.reserve(state.gemmRegions.size()); + for (const auto &entry : state.gemmRegions) { + indices.push_back(entry.first); + } + std::sort(indices.begin(), indices.end()); + return indices; +} + +const QuantizedLoweringPlan *FindQuantizedLoweringPlan(const QuantizationModelState &state, + std::size_t opIndex, EQuantizedBackend backend) +{ + auto opIt = state.loweringPlans.find(opIndex); + if (opIt == state.loweringPlans.end()) + return nullptr; + auto backendIt = opIt->second.find(backend); + return backendIt == opIt->second.end() ? nullptr : &backendIt->second; +} + QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion ®ion, const std::string &weightStorageTensor) { - QuantizedLoweringPlan plan; - plan.backend = EQuantizedBackend::CPU; - plan.status = EQuantizedLoweringStatus::Baseline; - plan.reason = "CPU baseline lowering with packed pre-quantized weight storage"; + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::CPU, EQuantizedLoweringStatus::Baseline, + "CPU baseline lowering with packed pre-quantized weight storage", + "cpu_packed_weight_baseline"); plan.inputStorage = StorageTypeForQuantizedTensor(region.inputQuant); plan.weightStorage = StorageTypeForQuantizedTensor(region.weightQuant); plan.biasStorage = EQuantizedStorageType::FloatCarrier; plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; + plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; plan.weightStorageTensor = weightStorageTensor; plan.weightLayout = EQuantizedLayout::PackedCPU; - plan.consumedOperatorIndices = { region.inputQuantOpIndex, region.weightQuantOpIndex, region.gemmOpIndex, region.outputQuantOpIndex }; - if (region.biasQuantOpIndex) { - plan.consumedOperatorIndices.push_back(*region.biasQuantOpIndex); - } - std::sort(plan.consumedOperatorIndices.begin(), plan.consumedOperatorIndices.end()); - plan.preservesQuantizationSemantics = true; - plan.hasBaselineLowering = true; - plan.hasOptimizedLowering = false; - plan.isMetadataOnly = false; - plan.usesInt32Accumulator = true; - plan.usesPrequantizedWeights = true; - plan.suppressesGraphOperators = true; return plan; } @@ -109,57 +162,211 @@ QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend plan.biasStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; plan.accumulatorStorage = EQuantizedStorageType::UNDEFINED; plan.outputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.inputCarrierMode = preservesSemantics ? EQuantizedCarrierMode::Float : EQuantizedCarrierMode::UNDEFINED; + plan.outputMode = preservesSemantics ? EQuantizedOutputMode::ExactFakeQuantFloat : EQuantizedOutputMode::UNDEFINED; + plan.computeProfile = preservesSemantics ? EQuantizedComputeProfile::GenericRecognized : EQuantizedComputeProfile::UNDEFINED; + plan.capabilityTag = preservesSemantics ? "recognized_backend_unsupported" : "semantic_unsupported"; plan.preservesQuantizationSemantics = preservesSemantics; - plan.hasBaselineLowering = false; - plan.hasOptimizedLowering = false; plan.isMetadataOnly = preservesSemantics; - plan.usesInt32Accumulator = false; - plan.usesPrequantizedWeights = false; + plan.supportsPrequantizedInputCarrier = false; plan.suppressesGraphOperators = false; return plan; } QuantizedLoweringPlan MakeAlpakaFakeQuantPlan(const QuantizedGemmRegion ®ion) { - QuantizedLoweringPlan plan; - plan.backend = EQuantizedBackend::ALPAKA; - plan.status = EQuantizedLoweringStatus::Baseline; - plan.reason = "Alpaka fake-quant lowering over float carrier tensors"; + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Baseline, + "Alpaka fake-quant lowering over float carrier tensors", + "alpaka_fake_quant_baseline"); plan.inputStorage = EQuantizedStorageType::FloatCarrier; plan.weightStorage = EQuantizedStorageType::FloatCarrier; plan.biasStorage = EQuantizedStorageType::FloatCarrier; plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; + plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; plan.weightLayout = EQuantizedLayout::Plain; - plan.consumedOperatorIndices = { region.inputQuantOpIndex, region.weightQuantOpIndex, region.gemmOpIndex, region.outputQuantOpIndex }; - if (region.biasQuantOpIndex) { - plan.consumedOperatorIndices.push_back(*region.biasQuantOpIndex); - } - std::sort(plan.consumedOperatorIndices.begin(), plan.consumedOperatorIndices.end()); - plan.preservesQuantizationSemantics = true; - plan.hasBaselineLowering = true; - plan.hasOptimizedLowering = false; - plan.isMetadataOnly = false; - plan.usesInt32Accumulator = true; - plan.usesPrequantizedWeights = false; - plan.suppressesGraphOperators = true; return plan; } -void CheckQuantInfo(const QuantizationInfo &info, const std::string &role, bool requireSigned, - std::vector &reasons) +struct QuantizedGemmCublasLtCapability { + bool optimized = false; + EQuantizedComputeProfile profile = EQuantizedComputeProfile::GenericRecognized; + std::string tag = "recognized_not_cublaslt_optimized"; + std::string reason; + QuantizedMatMulShapePolicy shapePolicy; +}; + +void AddCapabilityReason(std::vector &reasons, std::string reason) +{ + reasons.push_back(std::move(reason)); +} + +constexpr std::size_t kCublasLtInt8Alignment = 16; +constexpr std::size_t kCublasLtMinOptimizedMacs = 1'000'000; +constexpr double kCublasLtPaddingCandidateMaxWorkRatio = 1.50; + +std::size_t RoundUpToMultiple(std::size_t value, std::size_t multiple) +{ + if (multiple == 0 || value == 0) + return value; + return ((value + multiple - 1) / multiple) * multiple; +} + +bool IsAlignedTo(std::size_t value, std::size_t multiple) +{ + return multiple != 0 && (value % multiple) == 0; +} + +QuantizedMatMulShapePolicy MakeCublasLtShapePolicy(std::size_t m, std::size_t k, std::size_t n) { - if (!IsScalarPerTensor(info)) { - reasons.push_back(role + " quantization is not scalar per-tensor"); + QuantizedMatMulShapePolicy policy; + policy.logicalM = m; + policy.logicalK = k; + policy.logicalN = n; + policy.physicalM = RoundUpToMultiple(m, kCublasLtInt8Alignment); + policy.physicalK = RoundUpToMultiple(k, kCublasLtInt8Alignment); + policy.physicalN = RoundUpToMultiple(n, kCublasLtInt8Alignment); + + policy.logicalMacs = m * k * n; + policy.physicalMacs = policy.physicalM * policy.physicalK * policy.physicalN; + policy.minimumOptimizedMacs = kCublasLtMinOptimizedMacs; + policy.belowMinimumWork = policy.logicalMacs < policy.minimumOptimizedMacs; + policy.paddingWorkRatio = policy.logicalMacs > 0 ? static_cast(policy.physicalMacs) / + static_cast(policy.logicalMacs) : 1.0; + + std::ostringstream reason; + reason << "logical M/K/N=" << policy.logicalM << "/" << policy.logicalK << "/" << policy.logicalN + << ", physical M/K/N=" << policy.physicalM << "/" << policy.physicalK << "/" << policy.physicalN + << ", logical MACs=" << policy.logicalMacs + << ", physical MACs=" << policy.physicalMacs + << ", minimum optimized MACs=" << policy.minimumOptimizedMacs + << ", padding work ratio=" << policy.paddingWorkRatio; + + if (IsAlignedTo(m, kCublasLtInt8Alignment) && IsAlignedTo(k, kCublasLtInt8Alignment) && + IsAlignedTo(n, kCublasLtInt8Alignment)) { + if (policy.belowMinimumWork) { + policy.policy = EQuantizedShapePolicy::ExactTooSmall; + policy.reason = "exact cuBLASLt int8 shape below minimum optimized work threshold; " + reason.str(); + } else { + policy.policy = EQuantizedShapePolicy::Exact; + policy.reason = "exact cuBLASLt int8 shape; " + reason.str(); + } + } else if (policy.paddingWorkRatio <= kCublasLtPaddingCandidateMaxWorkRatio) { + policy.policy = EQuantizedShapePolicy::PaddedCandidate; + policy.reason = "padded cuBLASLt candidate; " + reason.str(); + } else { + policy.policy = EQuantizedShapePolicy::Fallback; + policy.reason = "padding too expensive for cuBLASLt candidate; " + reason.str(); } - if (info.bitWidth == 0 || info.bitWidth > 8) { - reasons.push_back(role + " bit width is not in the initially supported range [1, 8]"); + return policy; +} + +QuantizedGemmCublasLtCapability AssessCublasLtQuantizedGemmCapability( + const QuantizedGemmRegion ®ion, + const std::vector &inputShape, + const std::vector &weightShape) +{ + QuantizedGemmCublasLtCapability capability; + std::vector semanticReasons; + + if (inputShape.size() != 2) + AddCapabilityReason(semanticReasons, "input rank is not 2"); + if (weightShape.size() != 2) + AddCapabilityReason(semanticReasons, "weight rank is not 2"); + + if (inputShape.size() == 2 && weightShape.size() == 2) { + const auto m = inputShape[0]; + const auto k = inputShape[1]; + const auto n = weightShape[0]; + const auto weightK = weightShape[1]; + if (m == 0 || n == 0 || k == 0) + AddCapabilityReason(semanticReasons, "M, N, and K must be nonzero"); + if (k != weightK) + AddCapabilityReason(semanticReasons, "input K does not match weight K"); + if (m != 0 && n != 0 && k != 0 && k == weightK) + capability.shapePolicy = MakeCublasLtShapePolicy(m, k, n); + } + + if (region.inputQuant.bitWidth != 8) + AddCapabilityReason(semanticReasons, "input bit width is not 8"); + if (region.weightQuant.bitWidth != 8) + AddCapabilityReason(semanticReasons, "weight bit width is not 8"); + if (region.outputQuant.bitWidth != 8) + AddCapabilityReason(semanticReasons, "output bit width is not 8"); + if (!region.inputQuant.isSigned) + AddCapabilityReason(semanticReasons, "input quantization is not signed int8"); + if (!region.weightQuant.isSigned) + AddCapabilityReason(semanticReasons, "weight quantization is not signed int8"); + if (region.inputQuant.zeroPoint != 0) + AddCapabilityReason(semanticReasons, "input zero point is not 0"); + if (region.weightQuant.zeroPoint != 0) + AddCapabilityReason(semanticReasons, "weight zero point is not 0"); + if (!IsScalarPerTensor(region.inputQuant)) + AddCapabilityReason(semanticReasons, "input quantization is not per-tensor scalar"); + if (!IsScalarPerTensor(region.weightQuant)) + AddCapabilityReason(semanticReasons, "weight quantization is not per-tensor scalar"); + if (!IsScalarPerTensor(region.outputQuant)) + AddCapabilityReason(semanticReasons, "output quantization is not per-tensor scalar"); + + if (!semanticReasons.empty()) { + capability.shapePolicy.policy = EQuantizedShapePolicy::Unsupported; + capability.shapePolicy.reason = "cuBLASLt semantic requirements are not met"; + capability.reason = JoinReasons(semanticReasons); + capability.tag = "cublaslt_i8i8_semantic_unsupported"; + return capability; } - if (requireSigned && !info.isSigned) { - reasons.push_back(role + " quantization is not signed"); + + capability.profile = EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; + if (capability.shapePolicy.policy == EQuantizedShapePolicy::Exact) { + capability.optimized = true; + capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_exact"; + capability.reason = "cuBLASLt optimized signed-int8 symmetric per-tensor rank-2 exact-shape GEMM; " + + capability.shapePolicy.reason; + } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::ExactTooSmall) { + capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_exact_too_small"; + capability.reason = "cuBLASLt exact-shape execution is legal but below the minimum optimized work threshold; " + + capability.shapePolicy.reason; + } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::PaddedCandidate) { + capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_padded_candidate"; + capability.reason = "cuBLASLt padded execution is a candidate but is not implemented in this lowering; " + + capability.shapePolicy.reason; + } else { + capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_shape_fallback"; + capability.reason = capability.shapePolicy.reason.empty() ? + "cuBLASLt shape policy is unavailable" : capability.shapePolicy.reason; } - if (info.zeroPoint != 0) { - reasons.push_back(role + " zero point is not 0"); + return capability; +} + + +QuantizedLoweringPlan MakeAlpakaCublasLtCorePlan(const QuantizedGemmRegion ®ion, + const std::string &weightStorageTensor, + const QuantizedGemmCublasLtCapability &capability) +{ + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Optimized, + capability.reason, capability.tag); + plan.inputStorage = StorageTypeForQuantizedTensor(region.inputQuant); + plan.weightStorage = StorageTypeForQuantizedTensor(region.weightQuant); + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = StorageTypeForQuantizedTensor(region.outputQuant); + plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputMode = EQuantizedOutputMode::Quantized; + plan.computeProfile = EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; + plan.shapePolicy = capability.shapePolicy; + plan.weightStorageTensor = weightStorageTensor; + plan.weightLayout = EQuantizedLayout::PlainDevice; + plan.supportsPrequantizedInputCarrier = true; + return plan; +} + +void CheckQuantInfo(const QuantizationInfo &info, const std::string &role, bool, + std::vector &reasons) +{ + if (info.bitWidth == 0 || info.bitWidth > 8) { + reasons.push_back(role + " bit width is not in the supported QONNX fake-quant range [1, 8]"); } if (info.scale <= 0.0 || !std::isfinite(info.scale)) { reasons.push_back(role + " scale is not positive and finite"); @@ -200,40 +407,6 @@ const QuantizationInfo & RModel::GetQuantizationInfo(const std::string & tensor_ throw std::runtime_error("SOFIE tensor [" + clean_name + "] has no quantization information"); } - -bool RModel::HasQuantizedGemmRegion(std::size_t op_index) const -{ - return fQuantizationState.gemmRegions.find(op_index) != fQuantizationState.gemmRegions.end(); -} - -const QuantizedGemmRegion & RModel::GetQuantizedGemmRegion(std::size_t op_index) const -{ - auto it = fQuantizationState.gemmRegions.find(op_index); - if (it == fQuantizationState.gemmRegions.end()) { - throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no quantized Gemm information"); - } - return it->second; -} - -bool RModel::HasQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const -{ - auto opIt = fQuantizationState.loweringPlans.find(op_index); - return opIt != fQuantizationState.loweringPlans.end() && opIt->second.find(backend) != opIt->second.end(); -} - -const QuantizedLoweringPlan & RModel::GetQuantizedLoweringPlan(std::size_t op_index, EQuantizedBackend backend) const -{ - auto opIt = fQuantizationState.loweringPlans.find(op_index); - if (opIt == fQuantizationState.loweringPlans.end()) { - throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no quantized lowering plans"); - } - auto backendIt = opIt->second.find(backend); - if (backendIt == opIt->second.end()) { - throw std::runtime_error("SOFIE operator " + std::to_string(op_index) + " has no requested backend quantized lowering plan"); - } - return backendIt->second; -} - void RModel::RegisterQuantizedTensorStorage(QuantizedTensorStorage storage) { storage.storageTensor = UTILITY::Clean_name(storage.storageTensor); @@ -261,17 +434,6 @@ const QuantizedTensorStorage & RModel::GetQuantizedTensorStorage(const std::stri return it->second; } -std::vector RModel::GetQuantizedGemmOperatorIndices() const -{ - std::vector indices; - indices.reserve(fQuantizationState.gemmRegions.size()); - for (const auto &entry : fQuantizationState.gemmRegions) { - indices.push_back(entry.first); - } - std::sort(indices.begin(), indices.end()); - return indices; -} - void RModel::AnalyzeQuantizedRegions() { fQuantizationState.ClearDerivedAnalysis(); @@ -484,17 +646,58 @@ void RModel::AnalyzeQuantizedRegions() storage.layout = EQuantizedLayout::PackedCPU; storage.quantization = info.weightQuant; storage.shape = packedShape; + storage.residentBackend = EQuantizedBackend::CPU; storage.isConstant = true; - storage.isPersistent = true; - storage.isTransient = false; storage.isDeviceResident = false; - storage.byteSize = QuantizedStorageByteSize(storage.storageType, storage.shape); RegisterQuantizedTensorStorage(std::move(storage)); cpuPlan = MakeCPUPackedWeightBaselinePlan(info, storageTensor); } } + if (!info.weightSourceTensor.empty() && IsInitializedTensor(info.weightSourceTensor)) { + const auto deviceStorageTensor = info.weightSourceTensor + "_s17g3_plain_device_storage"; + const auto weightShape = GetTensorShape(info.weightSourceTensor); + const auto weightLength = ConvertShapeToLength(weightShape); + const float *weightData = fInitializedTensors.at(info.weightSourceTensor).data(); + + if (!IsInitializedTensor(deviceStorageTensor)) { + if (info.weightQuant.isSigned) { + AddConstantTensor(deviceStorageTensor, weightShape, QuantizeTensorToInt8(weightData, weightLength, info.weightQuant)); + } else { + AddConstantTensor(deviceStorageTensor, weightShape, QuantizeTensorToUInt8(weightData, weightLength, info.weightQuant)); + } + } + + QuantizedTensorStorage deviceStorage; + deviceStorage.logicalTensor = info.weightTensor; + deviceStorage.sourceTensor = info.weightSourceTensor; + deviceStorage.storageTensor = deviceStorageTensor; + deviceStorage.storageType = StorageTypeForQuantizedTensor(info.weightQuant); + deviceStorage.layout = EQuantizedLayout::PlainDevice; + deviceStorage.quantization = info.weightQuant; + deviceStorage.shape = weightShape; + deviceStorage.residentBackend = EQuantizedBackend::ALPAKA; + deviceStorage.isConstant = true; + deviceStorage.isDeviceResident = true; + RegisterQuantizedTensorStorage(std::move(deviceStorage)); + + try { + const auto inputShape = GetTensorShape(info.inputSourceTensor); + const auto capability = AssessCublasLtQuantizedGemmCapability(info, inputShape, weightShape); + if (capability.optimized) { + alpakaPlan = MakeAlpakaCublasLtCorePlan(info, deviceStorageTensor, capability); + } else { + alpakaPlan.reason += "; cuBLASLt optimized profile unavailable: " + capability.reason; + alpakaPlan.capabilityTag = capability.tag; + alpakaPlan.computeProfile = capability.profile; + alpakaPlan.shapePolicy = capability.shapePolicy; + } + } catch (const std::exception &) { + // Dynamic or unavailable input shapes keep the Alpaka fake-quant baseline plan. + } + } + auto &plans = fQuantizationState.loweringPlans[opIndex]; plans[EQuantizedBackend::CPU] = std::move(cpuPlan); plans[EQuantizedBackend::ALPAKA] = std::move(alpakaPlan); @@ -514,22 +717,40 @@ void RModel::AnalyzeQuantizedRegions() } } -void RModel::AddLoweredQuantizedGemmOperators(EQuantizedBackend backend) +void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) { - for (auto op_idx : GetQuantizedGemmOperatorIndices()) { - if (!HasQuantizedLoweringPlan(op_idx, backend)) + for (auto op_idx : QuantizedGemmOperatorIndices(fQuantizationState)) { + const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); + if (planPtr == nullptr) continue; auto *gemm = dynamic_cast *>(fOperators[op_idx].get()); if (!gemm) throw std::runtime_error("SOFIE quantized Gemm region is attached to a non-float Gemm operator"); - const auto &plan = GetQuantizedLoweringPlan(op_idx, backend); + const auto &plan = *planPtr; if (!IsQuantizedLoweringAvailable(plan.status)) continue; + const auto regionIt = fQuantizationState.gemmRegions.find(op_idx); + if (regionIt == fQuantizationState.gemmRegions.end()) + throw std::runtime_error("SOFIE quantized Gemm lowering plan has no matching region"); + const auto ®ion = regionIt->second; + if (backend == EQuantizedBackend::ALPAKA && plan.outputMode == EQuantizedOutputMode::Quantized) { + const auto outputType = region.outputQuant.isSigned ? ETensorType::INT8 : ETensorType::UINT8; + if (auto outputIt = fIntermediateTensorInfos.find(region.outputTensor); outputIt != fIntermediateTensorInfos.end()) { + outputIt->second.type = outputType; + } else if (auto readyIt = fReadyInputTensorInfos.find(region.outputTensor); readyIt != fReadyInputTensorInfos.end()) { + readyIt->second.type = outputType; + } else if (auto inputIt = fInputTensorInfos.find(region.outputTensor); inputIt != fInputTensorInfos.end()) { + inputIt->second.type = outputType; + } else if (auto dynamicIt = fDynamicTensorInfos.find(region.outputTensor); dynamicIt != fDynamicTensorInfos.end()) { + dynamicIt->second.type = outputType; + } + } + fLoweredOperators[op_idx] = std::make_unique( - GetQuantizedGemmRegion(op_idx), plan, MakeQuantizedGemmCodegenContext(*gemm)); + region, plan, MakeQuantizedGemmCodegenContext(*gemm)); if (plan.suppressesGraphOperators) { for (auto consumedOpIndex : plan.consumedOperatorIndices) { @@ -540,15 +761,21 @@ void RModel::AddLoweredQuantizedGemmOperators(EQuantizedBackend backend) } } -void RModel::AddQuantizedGeneratedHeaders() +void RModel::AddQuantizedGeneratedHeaders(EQuantizedBackend backend) { - for (auto op_idx : GetQuantizedGemmOperatorIndices()) { - if (!HasQuantizedLoweringPlan(op_idx, EQuantizedBackend::CPU)) + for (auto op_idx : QuantizedGemmOperatorIndices(fQuantizationState)) { + const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); + if (planPtr == nullptr) continue; - const auto &plan = GetQuantizedLoweringPlan(op_idx, EQuantizedBackend::CPU); - if (IsQuantizedLoweringAvailable(plan.status) && plan.suppressesGraphOperators && - plan.usesPrequantizedWeights && plan.weightLayout == EQuantizedLayout::PackedCPU) { - AddNeededCustomHeader("SOFIE/SOFIE_QuantizedRuntime.hxx"); + const auto &plan = *planPtr; + if (!IsQuantizedLoweringAvailable(plan.status) || !plan.suppressesGraphOperators) + continue; + if (backend == EQuantizedBackend::CPU && QuantizedPlanUsesPrequantizedWeights(plan) && + plan.weightLayout == EQuantizedLayout::PackedCPU) { + AddNeededCustomHeader("SOFIE/SOFIE_Quantized.hxx"); + } + if (backend == EQuantizedBackend::ALPAKA && IsOptimizedQuantizedAlpakaPlainDevicePlan(plan)) { + AddNeededCustomHeader("SOFIE/SOFIE_QuantizedAlpaka.hxx"); } } } From 543047f8605cad0bc649a74a9cd32981e17c9344 Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Mon, 6 Jul 2026 10:26:31 +0200 Subject: [PATCH 3/6] test: quantized GEMM GPU --- core/inc/SOFIE/ROperator_QuantizedGemm.hxx | 7 +- core/inc/SOFIE/RQuantization.hxx | 35 +- core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx | 9 +- core/src/RModel_ALPAKA.cxx | 3 +- core/src/RModel_Quantization.cxx | 67 +- test/CMakeLists.txt | 1 + .../TestCustomModelsFromONNXForAlpakaCuda.cxx | 41 + test/input_models/QONNX_QuantGemm.onnx | Bin 0 -> 66532 bytes .../references/QONNX_QuantGemm.ref.hxx | 1032 ++++++++++++++++ .../references/QONNX_QuantGemm_input.ref.hxx | 1035 +++++++++++++++++ 10 files changed, 2196 insertions(+), 34 deletions(-) create mode 100644 test/input_models/QONNX_QuantGemm.onnx create mode 100644 test/input_models/references/QONNX_QuantGemm.ref.hxx create mode 100644 test/input_models/references/QONNX_QuantGemm_input.ref.hxx diff --git a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx index c1ff230..af2c504 100644 --- a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx +++ b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx @@ -64,6 +64,8 @@ inline const char *QuantizedCudaInputCarrierName(EQuantizedCarrierMode mode) return "Int8"; case EQuantizedCarrierMode::Float: return "Float"; + case EQuantizedCarrierMode::UInt8: + throw std::runtime_error("SOFIE quantized CUDA GEMM currently supports signed int8 input carriers only"); default: throw std::runtime_error("SOFIE quantized CUDA GEMM received unsupported input carrier mode"); } @@ -337,11 +339,6 @@ inline std::string GenerateFusedQuantizedGemmCublasLtCoreLaunch(std::string opNa out << " params_quantizedGemm_" << opName << ".epilogueMode = SOFIE::EQuantizedCudaEpilogueMode::" << planEpilogueMode << ";\n"; out << " params_quantizedGemm_" << opName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::" << planInputCarrier << ";\n"; out << " params_quantizedGemm_" << opName << ".outputCarrier = SOFIE::EQuantizedCudaOutputCarrier::" << INTERNAL::QuantizedCudaOutputCarrierName(region.outputQuant) << ";\n"; - if (plan.supportsPrequantizedInputCarrier) { - out << "#ifdef SOFIE_QUANTIZED_GEMM_PREQUANTIZED_INPUT\n"; - out << " params_quantizedGemm_" << opName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::Int8;\n"; - out << "#endif\n"; - } out << " params_quantizedGemm_" << opName << ".weightType = SOFIE::EQuantizedCudaWeightType::" << (region.weightQuant.isSigned ? "Int8" : "UInt8") << ";\n"; out << " SOFIE::QuantizedGemmCudaLt_Call(quantizedGemmCudaLtState_" << opName << ", alpaka::getNativeHandle(queue)" diff --git a/core/inc/SOFIE/RQuantization.hxx b/core/inc/SOFIE/RQuantization.hxx index 1398b6b..de4462b 100644 --- a/core/inc/SOFIE/RQuantization.hxx +++ b/core/inc/SOFIE/RQuantization.hxx @@ -199,6 +199,27 @@ inline bool IsPhysicalQuantizedStorage(EQuantizedStorageType type) type == EQuantizedStorageType::Int32Accumulator; } +inline EQuantizedStorageType QuantizedStorageTypeForCarrier(const QuantizationInfo &info) +{ + return info.isSigned ? EQuantizedStorageType::Int8 : EQuantizedStorageType::UInt8; +} + +inline EQuantizedCarrierMode QuantizedCarrierModeForStorage(EQuantizedStorageType type) +{ + switch (type) { + case EQuantizedStorageType::FloatCarrier: + return EQuantizedCarrierMode::Float; + case EQuantizedStorageType::Int8: + return EQuantizedCarrierMode::Int8; + case EQuantizedStorageType::UInt8: + return EQuantizedCarrierMode::UInt8; + case EQuantizedStorageType::Int32Accumulator: + return EQuantizedCarrierMode::Int32Accumulator; + default: + return EQuantizedCarrierMode::UNDEFINED; + } +} + struct QuantizedLoweringPlan { EQuantizedBackend backend = EQuantizedBackend::UNDEFINED; EQuantizedLoweringStatus status = EQuantizedLoweringStatus::UNDEFINED; @@ -222,7 +243,6 @@ struct QuantizedLoweringPlan { std::vector consumedOperatorIndices; bool preservesQuantizationSemantics = false; bool isMetadataOnly = false; - bool supportsPrequantizedInputCarrier = false; bool suppressesGraphOperators = false; }; @@ -236,6 +256,19 @@ inline bool QuantizedPlanUsesPrequantizedWeights(const QuantizedLoweringPlan &pl return !plan.weightStorageTensor.empty(); } +inline bool QuantizedPlanExposesQuantizedInputCarrier(const QuantizedLoweringPlan &plan) +{ + return plan.inputCarrierMode == EQuantizedCarrierMode::Int8 || + plan.inputCarrierMode == EQuantizedCarrierMode::UInt8; +} + +inline bool QuantizedPlanExposesQuantizedOutputCarrier(const QuantizedLoweringPlan &plan) +{ + return plan.outputMode == EQuantizedOutputMode::Quantized && + (plan.outputStorage == EQuantizedStorageType::Int8 || + plan.outputStorage == EQuantizedStorageType::UInt8); +} + inline bool IsOptimizedQuantizedPlainDevicePlan(const QuantizedLoweringPlan &plan) { return IsQuantizedLoweringOptimized(plan.status) && QuantizedPlanUsesPrequantizedWeights(plan) && diff --git a/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx index 4d8a0ce..5ce23ee 100644 --- a/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx +++ b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx @@ -91,7 +91,7 @@ inline void QuantizedGemmCudaLt_SetProfiling(bool) {} inline bool QuantizedGemmCudaLt_ProfilingEnabled() { return false; } inline QuantizedGemmCudaLtProfile QuantizedGemmCudaLt_GetLastProfile() { return {}; } -inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &, QuantizedGemmCudaStream, void *, const float *, +inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &, QuantizedGemmCudaStream, void *, const void *, const void *, const float *, const QuantizedGemmCudaLtParams &) { throw std::runtime_error("SOFIE cuBLASLt quantized GEMM path was selected, but SOFIE_USE_CUBLASLT is not enabled"); @@ -805,7 +805,7 @@ private: }; inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedGemmCudaStream stream, - void *output, const float *input, const void *weight, const float *bias, + void *output, const void *input, const void *weight, const float *bias, const QuantizedGemmCudaLtParams ¶ms) { if (output == nullptr || input == nullptr || weight == nullptr) { @@ -844,11 +844,12 @@ inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedG constexpr int threads = 256; const std::int8_t *inputQuantized = nullptr; if (effectiveParams.inputCarrier == EQuantizedCudaInputCarrier::Int8) { - inputQuantized = reinterpret_cast(input); + inputQuantized = static_cast(input); } else { const int inputBlocks = static_cast((inputElements + threads - 1) / threads); + const float *inputFloat = static_cast(input); INTERNAL::QuantizedGemmCudaQuantizeInputKernel<<>>( - input, state.InputQuantizedBuffer(), inputElements, effectiveParams.inputScale, effectiveParams.inputZeroPoint, + inputFloat, state.InputQuantizedBuffer(), inputElements, effectiveParams.inputScale, effectiveParams.inputZeroPoint, effectiveParams.inputQMin, effectiveParams.inputQMax); INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaQuantizeInputKernel launch"); inputQuantized = state.InputQuantizedBuffer(); diff --git a/core/src/RModel_ALPAKA.cxx b/core/src/RModel_ALPAKA.cxx index e1ce168..542190a 100644 --- a/core/src/RModel_ALPAKA.cxx +++ b/core/src/RModel_ALPAKA.cxx @@ -298,7 +298,8 @@ std::string RModel::GenerateInferSignature_GPU_ALPAKA(bool isdecl) { if (type == ETensorType::DOUBLE) return "BufD1D"; if (type == ETensorType::INT32) return "BufI321D"; if (type == ETensorType::INT64) return "BufI641D"; - if (type == ETensorType::BOOL) return "BufUI81D"; + if (type == ETensorType::INT8) return "BufI81D"; + if (type == ETensorType::BOOL || type == ETensorType::UINT8) return "BufUI81D"; throw std::runtime_error("sofie: input tensor " + name + " is of a data type which is not yet supported."); }; diff --git a/core/src/RModel_Quantization.cxx b/core/src/RModel_Quantization.cxx index c11f8ce..dc6c22a 100644 --- a/core/src/RModel_Quantization.cxx +++ b/core/src/RModel_Quantization.cxx @@ -35,9 +35,20 @@ bool IsScalarPerTensor(const QuantizationInfo &info) return info.granularity == EQuantizationGranularity::PerTensor && info.axis == -1; } -EQuantizedStorageType StorageTypeForQuantizedTensor(const QuantizationInfo &info) +ETensorType TensorTypeForQuantizedStorage(EQuantizedStorageType storage) { - return info.isSigned ? EQuantizedStorageType::Int8 : EQuantizedStorageType::UInt8; + switch (storage) { + case EQuantizedStorageType::FloatCarrier: + return ETensorType::FLOAT; + case EQuantizedStorageType::Int8: + return ETensorType::INT8; + case EQuantizedStorageType::UInt8: + return ETensorType::UINT8; + case EQuantizedStorageType::Int32Accumulator: + return ETensorType::INT32; + default: + throw std::runtime_error("SOFIE quantized lowering plan has no physical tensor type for this storage"); + } } std::vector QuantizeTensorToInt8(const float *data, std::size_t length, const QuantizationInfo &info) @@ -137,8 +148,8 @@ QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::CPU, EQuantizedLoweringStatus::Baseline, "CPU baseline lowering with packed pre-quantized weight storage", "cpu_packed_weight_baseline"); - plan.inputStorage = StorageTypeForQuantizedTensor(region.inputQuant); - plan.weightStorage = StorageTypeForQuantizedTensor(region.weightQuant); + plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); + plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); plan.biasStorage = EQuantizedStorageType::FloatCarrier; plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; plan.outputStorage = EQuantizedStorageType::FloatCarrier; @@ -168,7 +179,6 @@ QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend plan.capabilityTag = preservesSemantics ? "recognized_backend_unsupported" : "semantic_unsupported"; plan.preservesQuantizationSemantics = preservesSemantics; plan.isMetadataOnly = preservesSemantics; - plan.supportsPrequantizedInputCarrier = false; plan.suppressesGraphOperators = false; return plan; } @@ -347,18 +357,17 @@ QuantizedLoweringPlan MakeAlpakaCublasLtCorePlan(const QuantizedGemmRegion ®i { auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Optimized, capability.reason, capability.tag); - plan.inputStorage = StorageTypeForQuantizedTensor(region.inputQuant); - plan.weightStorage = StorageTypeForQuantizedTensor(region.weightQuant); + plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); + plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); plan.biasStorage = EQuantizedStorageType::FloatCarrier; plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; - plan.outputStorage = StorageTypeForQuantizedTensor(region.outputQuant); - plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputStorage = QuantizedStorageTypeForCarrier(region.outputQuant); + plan.inputCarrierMode = QuantizedCarrierModeForStorage(plan.inputStorage); plan.outputMode = EQuantizedOutputMode::Quantized; plan.computeProfile = EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; plan.shapePolicy = capability.shapePolicy; plan.weightStorageTensor = weightStorageTensor; plan.weightLayout = EQuantizedLayout::PlainDevice; - plan.supportsPrequantizedInputCarrier = true; return plan; } @@ -642,7 +651,7 @@ void RModel::AnalyzeQuantizedRegions() storage.logicalTensor = info.weightTensor; storage.sourceTensor = info.weightSourceTensor; storage.storageTensor = storageTensor; - storage.storageType = StorageTypeForQuantizedTensor(info.weightQuant); + storage.storageType = QuantizedStorageTypeForCarrier(info.weightQuant); storage.layout = EQuantizedLayout::PackedCPU; storage.quantization = info.weightQuant; storage.shape = packedShape; @@ -673,7 +682,7 @@ void RModel::AnalyzeQuantizedRegions() deviceStorage.logicalTensor = info.weightTensor; deviceStorage.sourceTensor = info.weightSourceTensor; deviceStorage.storageTensor = deviceStorageTensor; - deviceStorage.storageType = StorageTypeForQuantizedTensor(info.weightQuant); + deviceStorage.storageType = QuantizedStorageTypeForCarrier(info.weightQuant); deviceStorage.layout = EQuantizedLayout::PlainDevice; deviceStorage.quantization = info.weightQuant; deviceStorage.shape = weightShape; @@ -719,6 +728,24 @@ void RModel::AnalyzeQuantizedRegions() void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) { + auto setKnownTensorType = [this](const std::string &tensorName, ETensorType type) { + if (auto it = fIntermediateTensorInfos.find(tensorName); it != fIntermediateTensorInfos.end()) { + it->second.type = type; + return; + } + if (auto it = fReadyInputTensorInfos.find(tensorName); it != fReadyInputTensorInfos.end()) { + it->second.type = type; + return; + } + if (auto it = fInputTensorInfos.find(tensorName); it != fInputTensorInfos.end()) { + it->second.type = type; + return; + } + if (auto it = fDynamicTensorInfos.find(tensorName); it != fDynamicTensorInfos.end()) { + it->second.type = type; + } + }; + for (auto op_idx : QuantizedGemmOperatorIndices(fQuantizationState)) { const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); if (planPtr == nullptr) @@ -736,17 +763,11 @@ void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) if (regionIt == fQuantizationState.gemmRegions.end()) throw std::runtime_error("SOFIE quantized Gemm lowering plan has no matching region"); const auto ®ion = regionIt->second; - if (backend == EQuantizedBackend::ALPAKA && plan.outputMode == EQuantizedOutputMode::Quantized) { - const auto outputType = region.outputQuant.isSigned ? ETensorType::INT8 : ETensorType::UINT8; - if (auto outputIt = fIntermediateTensorInfos.find(region.outputTensor); outputIt != fIntermediateTensorInfos.end()) { - outputIt->second.type = outputType; - } else if (auto readyIt = fReadyInputTensorInfos.find(region.outputTensor); readyIt != fReadyInputTensorInfos.end()) { - readyIt->second.type = outputType; - } else if (auto inputIt = fInputTensorInfos.find(region.outputTensor); inputIt != fInputTensorInfos.end()) { - inputIt->second.type = outputType; - } else if (auto dynamicIt = fDynamicTensorInfos.find(region.outputTensor); dynamicIt != fDynamicTensorInfos.end()) { - dynamicIt->second.type = outputType; - } + if (QuantizedPlanExposesQuantizedInputCarrier(plan)) { + setKnownTensorType(region.inputSourceTensor, TensorTypeForQuantizedStorage(plan.inputStorage)); + } + if (QuantizedPlanExposesQuantizedOutputCarrier(plan)) { + setKnownTensorType(region.outputTensor, TensorTypeForQuantizedStorage(plan.outputStorage)); } fLoweredOperators[op_idx] = std::make_unique( diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 12f19b1..72fd8b9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -148,6 +148,7 @@ if (ENABLE_ALPAKA_TESTS) TestCustomModelsFromONNXForAlpakaCuda PRIVATE ALPAKA_ACC_GPU_CUDA_ENABLED ALPAKA_HAS_STD_ATOMIC_REF + SOFIE_USE_CUBLASLT ) target_compile_options( diff --git a/test/TestCustomModelsFromONNXForAlpakaCuda.cxx b/test/TestCustomModelsFromONNXForAlpakaCuda.cxx index fccacbe..e3ca92a 100644 --- a/test/TestCustomModelsFromONNXForAlpakaCuda.cxx +++ b/test/TestCustomModelsFromONNXForAlpakaCuda.cxx @@ -44,6 +44,10 @@ #include "Linear_64_FromONNX_GPU_ALPAKA.hxx" #include "input_models/references/Linear_64.ref.hxx" +#include "QONNX_QuantGemm_FromONNX_GPU_ALPAKA.hxx" +#include "input_models/references/QONNX_QuantGemm_input.ref.hxx" +#include "input_models/references/QONNX_QuantGemm.ref.hxx" + #include "AddBroadcast1_FromONNX_GPU_ALPAKA.hxx" #include "input_models/references/AddBroadcast1.ref.hxx" @@ -222,6 +226,43 @@ class SofieAlpakaTest : public ::testing::Test { }; +TEST_F(SofieAlpakaTest, QuantizedGemm) +{ + constexpr Idx inputSize = QONNX_QuantGemm_Input::input_size; + constexpr Idx outputSize = QONNX_QuantGemm_ExpectedOutput::output_size; + static_assert(inputSize == 128 * 128); + static_assert(outputSize == 128 * 128); + + auto input_h = alpaka::allocBuf(host, Ext1D::all(inputSize)); + std::int8_t *input_ptr = reinterpret_cast(alpaka::getPtrNative(input_h)); + for (Idx i = 0; i < inputSize; ++i) { + input_ptr[i] = QONNX_QuantGemm_Input::input[i]; + } + + auto input_d = alpaka::allocBuf(device, Ext1D::all(inputSize)); + alpaka::memcpy(queue, input_d, input_h); + alpaka::wait(queue); + + auto result_h = alpaka::allocBuf(host, Ext1D::all(outputSize)); + + { + SOFIE_QONNX_QuantGemm::Session model("QONNX_QuantGemm_FromONNX_GPU_ALPAKA.dat"); + auto result = model.infer(input_d); + alpaka::wait(queue); + cudaDeviceSynchronize(); + + alpaka::memcpy(queue, result_h, result); + alpaka::wait(queue); + } + + const auto *res_ptr = reinterpret_cast(alpaka::getPtrNative(result_h)); + const auto *correct = QONNX_QuantGemm_ExpectedOutput::output; + + for (Idx i = 0; i < outputSize; ++i) { + EXPECT_EQ(static_cast(res_ptr[i]), static_cast(correct[i])) << "i=" << i; + } +} + TEST_F(SofieAlpakaTest, Linear64) { constexpr float TOLERANCE = DEFAULT_TOLERANCE; diff --git a/test/input_models/QONNX_QuantGemm.onnx b/test/input_models/QONNX_QuantGemm.onnx new file mode 100644 index 0000000000000000000000000000000000000000..049d4485b6e49cd8afbf129dc833670177c7072d GIT binary patch literal 66532 zcmcF~`8!qN*S9%iipmgC2~nby>g;=+LX(OjQ$j?@R8b<8c@`N8m7&aJXfo}49cdIw zC`toL8Z>Fv^q%MYUeEPD&mZt!*ZE=ZYwvURT5F$s4WG~F7MGFJu(DXaZk4*1U!c3k zUY9_3H}%aH=H_2pFDR%p zQBuU!J@EgX^J5UW*TpYj<^PS=f1euvCnS<-f|C57mz)s%->wJ$FV}6SWwr0Oz&koP`7W z@GQHZ(QrtD+Q+dVv&Ium+Es90Q#Bcpwxtq`9Pf37Ah&0s6i;BOD6ho!EGpzo0{VOy ziYD!$W}I7ex%@&RdB2G&RhKfy-yFj=I|hiZKr0!1GM%S^gRDolCh@J51>?7yaKfK5 zwAc}VyEaI}i^*H@eXa?<-kApZE7##|>7Cf}sF6LoeLa!B7X`V~UC^xI7REdqXI-yr zK-n3J2_Zk}`Rk|1nPWa=MtwOW78^-_6l}o29AESh$bd$H6r8@`G(4~>#dST)xZWX8 z@v&V8L>If`-J7GV?}`|9@()e?eR?8y+_i-nQ!K;rs5$V>c?mQOq~S+Z38-C~%tQ-X z!XC>dAZjVu6exL*E|};AMGpC-Fds1~R0Ay6Xy`q~mYvt0! zcx|3w&<=8?Et%eX(vHV33&W6cXVX`)W+Kp}jDpY1$-ebtbl#>o2+aMRNG;>-E4YK1NQvW6P z`qsH(N6sLgsydEBCsNqALU+JK^c8XQ>VW4f!*T3UK6uF9r{`*#U?OuGPV`M;G8-hw zNY68Y_h6jaR5?cMj{=-GBmE-jri`2lv`IQ_chz8vnbU&dT3F6-wsejw?c-IqKZV z^ZZ0D8+Xx1R`c;hsu1nilTY8Be2Mi#T~OZhoVq;MMT4$vVtiSY7gO6t8+%WYiuhQX zQy#(4^yl>cC6-3GYO!Jl(}~Q&Bs_TIC^1}?MSl9uK%Mh)D7)eV6W^Nwo%)F&C1s5p zu1cb-$RrfK)k3BR&Y>~e&XDaYdC+vkhWLNEMKZ?9h`;|@Oe2fwhPBajV2&i(Y@x7P zdjgDY)Z^Jb+={qMhB}=#V#RwL@#_ytx88V0)yjo&>dtAP;@84j#HyeIIY;Nku3#r} z(@>^K0smS~g7a$oVDmpk-ks;l+~uK~s3x@zoLp;g##4P*cm5tHQoN8IF3zS~L&NE^ zQw&ToT#0sR5!7AlHeI;+9va8<7$rM3c4*T#`ar1~x4t`1y`{A1ISs9*(`d;|JC==7 zcOxKu+IrBMX9nMlKakG+Qh2|I10MoYVNY@i84RCFY@KF7&z(is=HWq2GPaP&glYIV zScMnuJr!GZ0V9`h139xV7_PYrC!B7BnqUQ9GMh}#H_XB6q{p}}*?*bu6*tt(pU-{vPl+IL-~ZMPYpmS`mq??EsT-ex(Bl~rooBDyWw(I z2sl-Ir#+7%tS{*&GUNcTbD)ks$H))s72?&3^4l0>zn z!cfj^56yEKBo}6SGkbNspfp|`UZoBZ%N1v#c=r`{TYm!+b(00F_+Yqk1hGFumZav~ zgkP@zNZrdku(L(16dEJOvVXBUivUc`!X$13oIXAWbIdfz{GNRBu%?D%tKW>$Rwl4$ zaFp)XQsX|Gg=7?6VmeJ2#Wp78{SVGfJuKQxUWiT10+SBtpZ{B(~~r zA$xO0HN>~xf$yd{RP#nB>~MUB2HmZ2F7FpPJ>LdR`+ia3WLa{gtpX|%T_CmL6uE!8 znSS5X!5A#a!Do6MRJLF%4ozsFy{>~~WlAXp=@+QCD<0Zzjx=o_>LN=XF2norNz|}E zpUz!%0h(z(mh5x@H|1e=bpAYO->Zqit$palb%BKXZYn)>fVgPr!1vcfD0Wg37j^Ha zhA+io*22kXI!-W+`VfBe{u)CIyoRwE`tFi%hDoT?0 zJ)SuCtt8?7TngB+2hJ}|fJq!43T+bPQTtJHJ=YxPcWl6CM^fU^Y#}7D~B?XBIVxVTjqnocMvB#E5WBZjTP(2?) zX4?CLr;{Oxd=-a%8)nisAN)YzsV`X<@*Zc-_J-iYQ}EQu({N4uEo~eRMbqE<;Q6x% z4K~Dp@zNRiYvE7g5O^I|2wh-Bc52|mehMyi+i|{Y3OyRW3nx3=V0EO-ncK=SP#&cM z+<)y@v+o5oukdGTl;VJsaGOq*UIe?(YS7f@m#Cqf9_O?rv55E1r$)5c60&T60@bJMvoG{}fJs7D-%RBBf`VAA{@a`zI zNgkr5?s~wxr2-SO@}O3DH#KaX$dxRcgB?yhTz{;Z%3UhQ-a>UaJNlXB2)5w0$BM)_ zY9IW)sERI?4=G+>hf1^6F#XVZEI*k}pE@RCwE>S=%#8&nUwzoEVMkVA6qL-J4AX1o zV4GYeiI<-Yn`h)u*DGR>^S}*X>?tNj+zIgg)?21SqMYt4aUqw(ToEOwqWVm(@ z+}~yft40f0H=$nA;xh|_es3UKrq9MCksK5)KTZ_b8v0tqfu;bN>1M-v%0}Q|0TWbIu*x0pX4x^R&X=+3t8S6iyL;0(BNqSaDhaT+@T<1|K}=By!??< z^*VtVg+=48u-7=s-A+26t%KcG#Ta#}kaj1RLt$?#M2rnET5c10@9!@lrkT57?t+wT16LDB2Zs34g~r%%j=UtZSWTy+Hs z55%C+rZ&=5QAkS{$KWq5J-j)40FCc;;_Z8LNpUcTiq#BKDw+=oBK7p{y~!Xa9giPV zj^hveVJea*gRT2dQ!fd5hAQ8On)#mi{98V%&;LV)<4?i&z^7nU5R0NILSVlB1>)2& zcEX)FwDplf?S)PFGA$1l{@xAtTRxJ2D1UUSz7J7Ru>j5oX!Es15IrRb8;8`vLeCt= z#@E11@r#^mI|}iBcQ{@Mc|}GFJE`63S~9!;6=rnYqv<+rbmff}*!nA(UX_`SyX?O( zBrY2Y=gsEkKPku5DFZOD@*9TEse;6YM$pMlg`nDrysg*#acloDJl*?&1lh(xZekMi z`T7~Mrtl&$&Q-?1qoE*sbqe>fNE{}u3dD`t0cd>49A|FVB;Sg@5sytWfd9hD-Ot%L zTS3`YHf&y^dF@gE@uLKs|lOmylanN*QDc4G73+RYkLy*eCs6*n^mj9k- zdDm0BGBwbCGDxo$h0(Adp;Ua}H*`%1gu$LljIVmg+M4vhqxTa@W8xHUVRi=Dofb=+ zbmHli8!9v;RgP}{qXxffPQv#~ne172aIFfr6|J^vxu{V6e2-HY&p|2Et%JX2?Eni&F9Y)Xn;gZ0+ zr_mzsI<0awg_A*F7({qYtkKNzHDx83k-<|;Tiw+a-_u0)kX z`nc=l71)<-iFdQ@Ve6O0cwpf(qQ0aIt4hYnqRmmb;6@hCUMr06?Ft}yaV=374Pru+ z)`L7x8K;P+!6*F^TskPkZ9E_V{$@K#@c{pd{c1+iWJRNy5rH=k5^&av$6(D-qGeMX zNWoTVbatM}HBynp$l01i+%6Kfu2trW23%*V;)dy7H8+}dREae3^YS|h5%M}R9P7koPQoc>-iWqDimad`iufz>|2LVoF5a5gE*;(X@>uAhm|hHc>-WhiV9Z zKM#KhC4qyhKEHo)C!1fGfO116^z6S&VybK~uPhzw7A?kY-z`BzE())H^n)tl=%(+N z9--<(9sFi7LUh;eMBe-mqR{?^T#K8;vwSbbTld2c2UR=KpVLVE`^D&B_F8PR*C)^Z z^$_P=As$&5h5y+}`F$!cM%vjEy8> zzl}ObD4CKUM!6(SBo8L8Py)AO+FVYcAl?gFhEsa?fYTfsoGiZ+9%^RcSmO{=tR~4l zxZnq)JNpBf=PAHdl~sW#pw(SyWvR7RXBKSDtQy7hF?Wn$uhm0s2X_* zV-GCBo^E4yUW+^oxytiayS#<3vMsFP7aQmreFl&E&qP=F83SN3#B2cl3(*B-w@4NNPm9Vgx6P0r6svfvCHTLhK>BA-_FFMpz;C~8~jXc zAFA>8e0D>-t$p}mJd0F^xj}>PO=e12CoavfE)oF8npUJ}C|AgoL2h zHiO>wkfVin#4!Am8*KViO}K5dfTy#Zj7~^}6RU2(GX;4RIV^&W4f~tcOwq!k__wU+ ztT^aiCW|9CeDLZcAtKV`1s16lO#x>^a4h;IncZ5+o(a55*K^fK!{~I#^A^WhPV-2q zn<%t9Ibx{pb3VrVmpj5#jxVCf-H|`u^gl7E29T_i3ea$gW@Rn}q@0`LF zE4lRG)^@OBd=4lnO$PF@X{^`_bKP% z=TQ|%_kV%eQX<4o)fT49T*a9W1z<^UDA{o$1A3EO@TRjYs@tr_BMW`dyUh(&*ja@_yiW4>t=F=E#4E81*v%8=UTS8e27eqai)oIif(=Qgc$H~ z!{rIjnRyA;IGi4iiMNFyWSJYP2aBL}LooCHdJ<$_cA`=t3t-CPOXwFHfpu~HaKCOo z>HK(%e!3Blx^`FT#lC9dw@r^e+dEDsy*faR<*cxFhYr^#=NtRjGqXua;Q_fYwxg-V zUX}Ym_7>T=))y}HWspK{ANX`VfMD51W?yG2uG3Y3UAdpJlrx>2ZAgLbZo$NYYlDWN z6S>z}0VbXL!aB@-NoNV?u!nakaSb;8z+j#dOkNvKRHwz^;hV=%DP0>ctLI~KeHp5phq-za5s$5K`+e-saQ4!a#Z^CU^m#9tAGiFPGI!wN>guadzWOfWi;NgmI z^xAC&v~jV=X;?;|`Zd64{VS^XW>ZtiyN_tppG(Hy7Q>$GHs;)_0a7_40_QLb7KuDy zJ(a51dpEpbuJvbnqWUK7IKk5W5|%`LOD5H0Yk>`rqK&Or!1zaQ_nG*JzyOUSBN5V^qw40ifVvc65>xoX5i_24ydn-)yA=vUy6 z1wr_4Ko^P*>haPuhKyEUL}+?M8L?hOE_Bp zlPS11N_PHjhxXf@loUyT@vYKI5+mi}qCqFY=j+kJ``&?wxC2>#xBl3zYWKL3PTp<{W#^wFAPVpvq zub9DoJDx%I=XAlwpInlE^As*9Or|zg*=YFZCb8Z9lNlBg;i)`4h>JB|Ldo`tC>%D2 zST93b`f3Z#8crsXb40L$tB1VkiInFlz}t3a1m8USfxH#TAU(*(>>Pwxae)Wq(EG(e zm1pwA-+aYS!F|l)9YLT~J`2L<@NjC!W%ei9!0GKzSyO8+<~z*64H{8!G_W2;e(NKX z>jgiPEb-N$WxTwuGLq$Uk0$QCgY7|=(Rzg-Y_l+fE$-RaeDn!9a+ODy&-89`$XN}i zxp#24>`bhlzKSalT!sO&=99|Y3|y3N10LLd;&W#$HfP(jr)5RJa7!NimH0riw^l+P zCxNV+lnEVy!{o=eRm9PMGANYF!Q+Yv#CS^>zRwS%D_u`Q`W+tQ{AxY^eBDJmZ=444 zw2PQ-(1sVKIXGL)5AU4zg$Oo)V4NB}v2Law(HzLq)uK$uVN{ad#^2Wd3P9h3B|+bjyQCr>C`h?i@tdw&F#20K=S1u!qd=oj`j&wy1gDQ$m7 z(YECg8>t?N{+}L_r#IZ7H%go4UU*9`{FH(6dl#T}elu~`$;PvDPQrngoz#g&&_(lt ziDzvuNo)E=4MOc`T(JaMU6_DVH}|8jx(F*ESHVm#QpBd6i}3KjM0)1oTw0U=hKJc=8e^Ce-0fzm>S8rU2eY7ICgm(t@`ml^CA33Xj~`iI$seF~>%qeRJO!-1SJC&Rsq$g_&2;QC$=%RcL3 z^RZ#7>oXU-ZyrOl;BHt~oB+2c*KlrYp8y5!b=3Ryp00l02zIyz-bf#&^9<_HPNxi# zW~Je@MiX4`wVo9!J=er*FeX=DUM9<>JIM5Jv$&jBqaO#`L9D`}a*Y4($F1Z&1zp_v;h;o8+w za49(oQpF-DQS*v5q+ysotj%@ZG(x{@3nv$rIib$p8IUmF4-9AQ!o#c<_@}PJX=(!K z^s9uOPYv+i*>EU&mQSM>KVaGoCjy;oft!wK$mW%8sGm6nY`Zro=CjIJ{GllpPo#3~~#`N#{@$;yyCzlP(_(}c3qIc(>*V0d>^0%rtpsLc5;+;^c4 zyN@=JCm(sNd*uMM~KLwXAJSG$3=7D>u0+vqbMfB-l ze{R|iOC%nW=Vj4oe|{MRnN+Zj(gO737XjFOE&$WG$zZH4z`eBj1k!D?{F;!14FkDk zgOCP@Sqwr$hBMkaex;|zji67oj*VUMlcY!w!5uLGJaQT{@}5D`^J0|MG>2Qg+C-rC zFZro4k(g+IB(7(EV(j8eM7GiudqOv3>Y)it`G;1f$Lj*!_WLdTQrrmHTLgH0L1MW5 za4Z_IGx1OJAlz1*%Y0bw2E!Kod{lh~t4fn_OP??OFe{c+tlNZ3hVGG26KzJZyMV;} zJPY;$n($rXJ&1;NQm>K;#70GsXP&A`jc#p&4TtYyCGo<%t16&aYmCOd%1|Ej28Q1z z!BgQ)^Fv0OkqQ9uk6sj%ZjLl z7-0avC%C|`Kf^b}aNXm>xI_jq^xJA!eC;k8^gLjq{%Vu&{c;#KWR2@YPSeC8OR!&N z4DP8c&Ff8~9i}G%8DF&bp3GeOyd7qxodz~349}nYNCf|iGri_dFtouRFPbYrqRYFc z{@Xzuk?{-^uL+>iV~Q{sdl)(-r@{RB)#OB@IMyFj9F8o4!Hle;mkRqQG)-0x+&|IeL zRx{`69VwWEJl+SS$fX?A^OPO!LO^!af(b2W-guyY+gFkyM8y0j8Ebw5;L;A zCK{ISl!OZ#ztG9v&uD7!Ovn>xWxSX5gZ5(rV@Yc8_|`IP*%^SZb!>2Vx+r&wZaA`H zsdzhRHbgGez=bjVyn1vBuk@EBJ8PpUtMW@4Aa$$~mE`2{Rm4rg8@)m{`rKtYpXShQ-u~3T&4En$ zZVILy0_Z!X0>}TQP$&0qF#foJT;G|)oWAOhZndXTl;=jI;+%;2g*0+q*8}(0_R(ol zjZ|~tOf*fLgDbSA;SRHYREk`L8V6pX?C4}1nsyQHtCir}^bkncI)Nv7GVPIYp>^ULHlh@_2ErE)Vay^I+rniw$6<ITaHzlbp*~# zdry?bGpH36re_+vmSxB^vIVWO+$#dpFxlLV>P}t;&sMF2n?^PecxVgRaaj%ac7BG5 zfyt01C_*QM$MExD78s5X0C!F@SnneE=c^E0F8IWGpHa`5!8N0Lr5y5fxg6+za$u3y zLLQkMLE|TX>0}Ez^b-|~~uRs8^ZL({2Bo(+my zWI$WS0$9}Q1rvlO!tgMQH#5J|h1=EX%(@87^lreL^)qm-YCdepI)T%Kor$|;IZS?j zjgdKclKx0dz_m|i;6IBrc43Mj&sy*|bKYHuYbz9m?NfN*6(Wv*d!k@tK{2R@K4+F| zZbSW{Cc5B`4Q?Cgrc!lj;9e4hRY!E;NSg#cu8P34k$AMQ$bpFdy^vfVNDoSWr6miY5VS3rX%8A+Fen z8%=+x58u2sQMxM_^NmEP>-W1CO z#6G8q-km;7rd}pkzye5$wWGSM7d)R#cz)YuA<^YKP07hQ{>Wy?_r)3XEb*~Tbw zvKiXD;~Oqn<&#DaJ>JW3QyQD!Mus@WbfuXEs3eBMH`5pJ>V*fY@#lHBr^iW>Q7i=c zj^W%brR2-s&c>|v3}}zsgLy;NV3=}%D$ThMZokrrtdclf{`QHScUXYRkM%HWFCPm$ zsK{gWzo7q@W@`8ECTtFqB~_+#aUxv?3%^Rjqx%Q2qND(n zsi4N!7wy^BOk>iVz*3?Bm7jkmeJ3k%dS4Li>CC_)jc}-P+lxLQ@;OgTYG~olLNv5V zg1Tkv;fdoo+dg|Mt2#l4 zCux(#psr;u#wx1A&e{!F9oPX4wM8^lWeFKGe@EV#Ttj7M6?$Dsfv>R{u;f}BDr86E zFH;}ro|*$9eMd2G;RjmoEQ2bEk|DJadWuuhVB{#SahgFlc~NZw0X`;RfoNZ^ir8TiE4VFCol-6Piv_L77We&@=x7yM2Kf z*stxQJ>HLStW}MsXr3a9vKHX~U7ct1E(F7Br$L}n5jhdw1kPW)X_k*3v%*k=$ZI5# z3PVrGXp3SK^rPu>!9e(KaRjDT9e~HK5@6^pfO%)Up(yANOj@MKRd0)Fy02kOe`?&M zD}PUgpj%?lec1;TYWm3MtQO9`th3Dg&9#Y6A2%p^sc=sPUd&?aU zZZBmXO?eBN0p9rbV>g{M;tN$?j=XhK+et*2EtsARgGZb0kh3bHaPZR`n)s5>8MJ3J zzx%>rW4a_%KQg3|m!069?GW9>k8$?R6;!SGG|b;K3la(=X=2Y&a8%}kWWausD)NS; zO)qSETHQfpQuFZF$pRWYs)=9kh=QSL4turbEIQf>!1QkMrnDF};#9s6hhnB;#?=Yj zg;|#{Db!v3C;-S;i1xE+RYF&vJm94!hyKxz%AnPb|KlQVnC-fj?fGh z2i*Vo3Y<**#RdiJ$6H1gRO#ts@JCgY-O-AxYTm=PucP$SNI7}ApMbKWS z!8I{E1WMluY1pYeurC$hKIthzw}3EW92p7CkvC|D%wBq;cq#EN@PV%OOzLpQ4S#?J z@6Ae0Hv3H)Oju@$D}C+)oX^C)*FA8JZWv(?XHv0E1(0!e5$3za;^&4Ilvj)g6Zz+G zG57=3i?;x?C`;lw)(J1N1)xBfLuMWo=ZXGR1+CO^Y&Kbg4_A#d&pr2p#&-@)@@fV~ z=LUJ&lFX>@P9uh@k)UC&4S9S$%o|5Vy7_S_&dyfA#L#18`MV7mdFlg=yst?Qbw3B2 zrK)6|b`G3-<4ulR){>^|SQMFh7Y>z{;ftyR^tf~x$_h8L{SzmWF&!HcY9+#baQ6f# zXKF)&fGd6s9L8BTES;*A3+DRk8cliANUT*meQQ-rv#xugz$#HV`a_7jRC*Fwv%7*i zZF41C&))%I=QgTn6#+I4B3x6?pY-FYSkmul0<(gn;mOJdFupqhyI(cI&%ZUe!lM)) zS=7U+)kdJbd?ESk^96mq4xoRlC^T*9C3B~l!n@Ip#G%sKupM@-iowXLbMS}=F&aCUV`uI`YP{$)u8yAxKH{IL z=8=Q2JtUdV9$iFJd$Vycvjn1VpJd-|yb4d1-HAl%9`;&594WMAFbN7EXiqWSWpk5R zf-!Xap&z96ZxgKbJ3{UEuYhwq0>FN!9tn8Q-+y9x3RgcYp#Jm#OsL*~Pr@8Hqsz5; z&)vgGzey$O@NOgzc8pM4bs28jsbmmV*bi(_Bqu3v5M@$6v#xK<>Aw{YaAVqXd^d|i zW|1S?)3pH)tkQbR(uh;fP1n0C{be{ze_jSq; z`R)|FH*)~5-oJ&9H5LIq+KB&R=3o(@13$7v3#vv#p_-im_KyXL<&-%PVY--f>wKrO zR(!qPg1NNBIR@?prD5Uq$MoOT%S@iN7oB$`nkF6dz^eyuG2fee$?`dl@ZR$ww3+R| zG1YUNNc(hfj;X}U%A+LROcfF)wUTn3i|B3N3O_E361B9WaMJZM^TtjVuD(LnU{w-% zo;U;Ozz`A1*TdEoC48LiJk+|0b1kxTh`d}r`*`Y6)VLMOlkHV1#qz#*-CF6G*r`s{KGfM%%VlyCs{$D?0%8X)op;(PqC0JY=n8g z{*mz>2AmF@!uqFfv|Uz%JDQn8cRc4{$g7o*J8uszY*Ay_z#@>Za)$J1;Jwmz!L?Fu zC{Zkn6*FGp#5uy)XSofJt%!r;V&-uFUOo;kOh;rUVv(Q*W0`drx8#X)TjHnVh;b2& z3{N5(cgxb@)+LZyc?z5_L{Q5PVV;EUR_qm>k5_k%uoD9l*qG1TfbBBGvkQuF`QZYR zu6Phf?J{xN1q1fJOa^v-y$cJMUf|CJ7BDw#GP7y#G`#yQ^o3cwo|}+6D2i9j zpRnq+ml%N?pXiskObFK9fxfx95WIaS&W!QLflU&;f3|l?!x&+QHVSbcsLO$=Tm*(F zu&^lc2WssKAeU!^GCu4CM#D)TY<*skk~%F=*ec8R2FF7~wk-sXSb}wo7;JD9h6l|Z zO|rFzf$S24?bcH;ty>qDJavGt`k|OQ$j7Fcc=#CdopOa16Sw;fD82PKnvqv*QSwrV z9nB}gvgx?XwT8Gmp9ACdsqoIi2Y7$xfUw$GP(~C9*@HIwLF41d$ zRKY>1mTDbrr*c!LFvYd9Q1CVi4IAv)9uYBG_iP#XMtr8HUd`pc78xg1Uk2c8IG>ZA zo=Fo$zY>YpFHrL~54S!lp;zW^!$nV4LRm}s~aS$!Fjw<{4j&!fb4!gSDE>x)A$%6z``9SSrX>6*S0lsxu@Y-i(1)qo?8Ofp3M z*%9>RoR{S2yqUD{@F}wJPYrZFsDtRI&LnPa9o1Q{4KE{)LjYfIt9)obNLJROzd|RK z=uIUbQ^CXL(uB8`8BVlv84a#$N;%|c|C?GS`WZK?OeYcz8pX(QJ;YvMt zVxo>iZ<|3eLkq)ezrfM%X3}=x1N~H{!wMW5X1=Qka%-13U{=s^3|c-NH-HI(e`cu;rHjo6IHWUjJmPq2J z$~m}Z_eWUs;UdhAaz%r`^&oyKhi%_jj;rbpkiXUGbkd7c;P5gS`qsLFgr^JIi0uKf z-vk1z^T6L~fUonlrT2vdu{_67~F2&yGqqO+3EP|epV^rLn=A>z?w!|7Uj=7<~GJg}yV<~Nb@kLUSw zaRgkm$|k1g_A?s4MDg9_PG;_ELKX{$;rs*L7=Nam4hlCw!fYPYYWyNr%hZ4|OD1!5 z6Y$i46?Rz{!vf>wc(1IJM*b+m#wa7SH7jL#<9w~ja3y`G-c26s=i%ND>p=ajJMuCr z>BabFNL)M@K6;4LC*~1E$cuyer$6Jxsb?ToZH!a;GY5~S#3Cobk|b{JU}e?IAYNh~ zy!}?kPC01Sq&OZ4rc*_D*WMez_1wE)>GTVaH7-FTP8jXFCIM3>jH7m{Bz+zyLB(6Y zl5e*NdQNSkG5I6_vnE}dOjbPO#g{w4t=sT@H#CY~aVwm_2^^2F#IcoA`W{f=U z@*M&<8AVdrcMvR$#31TRH=7@wMK1)(^YpYMP~y-*T68=XP6-@_g4?TMi?u!6)((e! z84JFSCX@_VIT2gGU!3SKX(V&*MX36^9~-S~QJvdMoH@3z@%nDMxYh&R+fZ zn+iHBR>74&hxxO#29z@zXd<@-mnm4{g8Mr3d%Q87{c;4p)t#qz1x?U)uphGbWRd(! z2B_K`4pB0Vxa5o^3>>M1#BKIi{%tO?FAamKH*e8bIeyS|csfrcT_3V9-o}3}?_lBr z4`_bV%;y%usd)D$sv9Z5RU1`?ySZ2CCBZhhRBr=2CFDUWnvVxnredL#Da~9Wf{W`j z>9g57w6fR_?vj7xV9`9x>SQ59bPXs(lws+EL45MN0K@KFhn_4haDs(#H1-v)R}o#h zT<|2;{44?cBXWHG_is9LQ8%0AxgLyEDwqqhW^j7#XFRZmuZ@efL3M`G?P&+`%_$|G zK}9bLc-oVp$Rzwr_rnXNd{SEH4CNM5AX52{b=LX>F>`N{yl8bY>gR=9d(`OO@#SoN zi3YUQ-6UR~H!xa00~;;E!B8%mHP%j~L^c6x-R)q}!AW4N^bB=uO3=>n1*5ZMIs_cz zf`#f7bk*lG3Llr#DGk@i@_&_Vo#PGMby11b8XPB9<9r?I@j_fFtWCDXw~^v6oiI7z z5&dfKi;2;T;iHK)iI}R$gm`B_={jrd40nXqwqK-s{~mJV{kg{VY$O_bFX4@9WuoWs z6f!&EDwOHTbDbUp;fq7z@N36JuE9x`Ra$!%eqKAw^bYn>r|Epm;Ee;;7Hva`wP85v zloco+)P$G9%P>4Xnof)2&(Xyb@ssl={+bePD%?B?*4OT)2UeXxwH`@csr6Od__!B; z@^uKOriwsq>lf-7*J938}Wvo7SC5p99X?oxK4H%9LIXR+r1FRz5HqWYyO^!X%?7w$OCM4 zf25;J{$b)|6=*G%08Z3>$S5@6W117WAumj_&0T(M^YCM>!bA%g1TI#W8Z413WBNb+Dcfs1PT{u0?jcgOyg|E*I!okZ( z6l>O_xOpL%AE^ePC(+O@c@9LYGC4HY3}+2^g7u9)qWW44HyHES9ejP=wRuRbjK0w( z%RgZ74(aH%5ESq>gn%=9QFHM_aNy{{opx22A9W1soQuKBX9;cL?>B-XzMif=3GU?! zavhd*;`nF*d6_1LPWpN9e;7LNxE#MQj0;6OrLB_D5Gh6Dea`(x3ze*Fq7r3Aq)6J7 ziiUPd3n`V7)N}5)($YW)g^EbY=7$h|&;NZsy`N{DbFS-qUH7|%C+R^AXH0l&P3W^qApQ4=H7yT63rE>i$c>{Nvc1BzO`y znSXcsTXw0;b@sm#E->%LO{#J>4CP8y(Qm~#(!XvS^n5CS%|;Uhzoid?W>z~kO%21% zo5sPll1nty&Il^v#;_Tq7|liv{-N0q@m-Dp-nAQG+hk|>Ptg`vPu?fU3%17ZW4{2` zZO3~n9)n77BJSUtLhLSw!6M;l0@Jq>NwPr`lYP8^_&j?`^ON4tEq!t@w@MXvNPd9h z2_fX(=NTx!I+sZ5tfi_mklYb!LC=wJn)jm|E}i4_RmlUSygLMI>LvI(;@;SLFr0j^ zEhqD|)rnTxY_L#V&A4YiCoK_PWHjzMeaiVxN=;jF!|aWa3Mue-ECxKgr@|BQ&G_QW zQnJB9f>F@?h?m}_lH+<+P&L>?-NpWx=bgO?SJ#{eJ@;tXVC+Lad|yKhjH{Scrdjmu zrf~Rh{yXgVnFCbfGd<$H4Hr3XhV#p=F{O8q+M8d;!NM36IT?(4v)0k}y(h7#AQ2wt zZ^cB{`6#pH9TOHV4AQoW%qoW}beWI>#k0lvE>6e5W=cFJoiGC(GZvf&iC~w$KlX6Z zrA$L>w9(GMO_j$X{Z%J3@@8)J`;0g6$$cG}BKiUb-LjdCOcmU9@&F!P9F5o#Y% zJ^4&K-=&d+ty^eN?{|o7e1o2E9N}@1B7d9OD7)>jBYx8igC}+G=+;pY8kTGgvr|1; zv9@FMNlQK1+&-S}&JSdz4-F8Ld23)xq9+`?p~(+YID(Jg1miF3KW6L|F8kw_%Uy?8 ztk((`^0{ImjARGE+6RZxR8yKiW7A~*+@m2-G*1DZ{YwU^zs9gG$Pd(-5XVbP@Qo(@ zAvQl0`4;b|pyPP~jGx>L2}k!)N$qH|Abuy97+oUg!d2k2>TBMSDqASvcqXwK2cb&t zJ>jbzLf-AeI5NACxQ$O`J0|6mnahvDaCRW`WXUufDTs!a%@XkEbOf9lTu)6JZ__Il zHwoQP$n1&IV{Z8pTZ$)o2*en>i{~J z$iWm*51b?Cj>V2gVM){lRBsN2r1{~H@iY)3tKzVFizAG$4Ttt=%V60`b!3M8VJwBk zXeCi9^IHp0v~wR$sD-Z+nswEUs!Oe$L4e1csU^5I|6DcH?*C7d`xV5Mk2#2BuGipx)E zTd6zbPdbe28=8so`PXO&zQj>Flq?()qKz(vs2x0rMFX~Qe&1De_NqI-D+4!7h1^YH@tI#vZD-VE4zkI}nQ^(6o55WW*m z9A;p zINVv&Lmz8#`84HERN7)9q`l3eeNLIw=xZ*@l2g#VZUVmbdR5Ig2!i71cW4$+N`ImY zRHo$OlGqKHT%SQ@b?qt5tfU9NwzApbXK-%CHq7U|AhDcL`fF_}w6`YFtvdBY;cyXH z^Y60(1;k9qUkV>?cH#AgY{nzUhk2X69R%ZpK4>5}M*Lo`1o@!@IQ2#@$n2ZSHhtHC zj!re8PSYT!Ya_?aQqp#zjeMHnj9ZN_!pj-r7~F2o-2Y>W**_XVA|M0!>N#W&4TjS* zCi8i<*2Mct7VI&3NwN%YH3c=6QI3xf)(^6@^;uVmhwnHq}j4 zWP4oa;g|7Wso1O@)LZ3`k1h8@(4|l;u8tw2->%RL4~}A)dttTHbTM3Yvx`|PT}=dA zG^pRN81}4qD!sJS1z%j#5|pJZ#4;6*3vfIH$)_dp|&L z4BTGt#yj~**ySRL`HQ~5(~)i{%MYb8lhbJ5Sd+bx@L zN$nV(sf%P?IwDBf$90`gwbXmqUWcd+IqS8M<>PVAG07-m>6a(*IK&AGS}% z0&R7u%TUGJE2H6JRyV`+rhB^DXiDA%kJgcG!5A(zEUwa61(k}oG2Z-|D$NABFws)BZ z4`F`quOJ9)i-3Uy16&ZI0Q0Mq`2nLqyH82t)eA<%^!69lQdkj|j>Rz@i2|tI;Q}t6 zJur89GqE~-2+qoz;MYg{VQl7ohQxp#jejLkVu++-*4 z5ANY|zMg#8X)T1xc!?gBD50+%($L0WH7(^B&XdBSFgGg+vgh|w$D9FJ?p_CrYt}JL z!5NZJCj&D5v!L_55?t}Q!PuQygl?Y3yzP&oh`U`IF3|l23va~{BU@caue(KM_nc%u zi};|CKn#of{qXv&gK+t*xWFf66&X}p4e!o5gOcT5RO#D{V%1lPE58dz+w4u+S%X5LEj%}2;FP75|hi#uCMUzit`edc_>p+1Ol{RP|n zg^8-vLK=~BlimD%78a!~#Z#ZFh|%HqMEpu9Rw|8=7l*y+7rF_I?M%@-d=b3%ui%}w zs=^ZAyOfvx9L4=QNNTGZ6%nsQ3(?)kU;7{4xIaibUCx8@em*QazZJH8-U)@Sk<8Rm zbJCVwfQdKD;i~aT*ySoq4d;zV;WzWZTYn9jnZ$rmy&@UVi9*ZFYfv2ah!`i!!=a{j z$h=ufD+p2as0P2kc&|CaA2CAKd&AXP9lCJw?Fh>mk=77N$gToP^_?(+!>U@lWY3vW_>4S=vp({o%jSvjw+m@)eb+qcj31=dGPOiKD>N) z5A*8Nm}7cB$%yGovfjW4Zq+RZ?bkbLDY*ze$Yq$uJs}SI2&&dGSak6>ac`bRXUyY6 z!JpUcp=}Rg?lC_iS!oaFd`-w4<%xJ_w?Dgq>+4xrE<|TV948KECNahDpF%`*D>|<2 zB>#H5SY1^oo|X9@`1e5)CWI-`JOB2wIudWF@w$5K&H4(Xf4Q^l&*@yZP&&@hUx2Hx zc+o|(%RuK_6tu3ZB*l42`G zZOQiDAGl*_4Stv0h_5D@fU9!^4s6S2YRfqG;ITD!s@lNUs)@KfawqzYFc5+4Ve2#z(0ldD6wnfzcI+U1r6M(^*D(Ib2C(0}FC*q#P$ zQYNr_nF6!o`claG6${H1^D&{T2dnQrB9>oSIKIyc4ony!nH5s_HDVC&+G_*n1yPr7 zU8;Lg8I5O(3dRnWQIiAtT!tjPsui{n*{=`Dd-vTmA^sqy{y7Qr-lUW1Bk6E#qCA;? z<1F=gp#s-_Ob`@pl|h>sfnb!?O!j~K$xPfoPQc_fvb!7tY3-Ii_5_&&J-dO-ZwrE( z^-b6-97(JHTEpSZ3$W7WBV%oJmq5aAa68sOL|xNS%_ap~E$tWu&P!<>^QUS_nJ_nG zIvMj{gGWN2g2>hZrqGw`k?H$FGp?;>Oy2a7j%%g#_=+ehYHkLj2cOXHVgsChv4#Fz zYXnAK@sNK_iB<(j3bfqk;yhawwn_arIU#!k0w?ujg8w~C=?G>*{)KaUiSuw<-AK^B z)tYbVzn)&6ILdtLI|m!@H^WJZHS~>bCw;j46gc@-!O1ZxCg0~WyqF%vR%ck?g3QCX zp@TC>9UoJPaaEufEChFICzAZ(`Jm%9hY#WDG(tNYcEk+edYy3W5HF>Fj3(o2-!7V# z@SAz(y&G#||4={c1YEA8L_W%8!Mwai-j(NT;OqSDxOew$T5n%Ter~Ej?UOI*k`q}N z`Qid~UNiyTeZ9fnFL%NS?maEE5G5T)y=a)_KZcXT@$$wC?12_V@cVrS*Trd*e-D0B z(d5~<-QyK+SMfsFVJyeL)Spiln(YU%1zD8nYx3_o-2?xqbb7@{kA!u|!^jC8ywW)t z^iwXAPJ?OWpvHg1y3>RF(kw^mOVMy+tO2Vo+rjhZF?N;fZP-xCCwDKsU^nia2YB z?<%40z!-gb3~U4(hxu0o9qV?YUZ6DiwXMaUiLq$5j2x%<2gJr_h^mVOL`R7|ESEunDA^9uE9K1S>Zb6{7*M%=bi3kA9QtXkV#$Pf>P zsoU=oeZy9G9~X>)ox7;J`WCo9K>(7QFT)L&-SD_#h^dw zco|mH@DiJ!7R-K`C`I}$9PpMw3U+Y(V+9R_Gr9`6L=97+`2;WB87_+Rk;j&r^dvKc(0n;I7!0{l*{Ouw>Epzed-vh)iY@tA(6#}t4 zb76iSz_Xic(D1SY${H&0g*<2RPabzh&*&vkl&en42T#M4sx~%U-x4!}pMmV$naHlT z#M_@c={W@zSX!+}zjC>L>&=%y=Ttm=UD8b+7-piJ-D42`vKp^F7o+Fj=A+4rG<@n- zN=SqT8Id^)?$T0N7N;jDnDhxg#Y}~6{UThE7fIWFoQZ0bIG#{=M4bZxUDl`5#?-|q zSU(#kHwZwkVj*i9ejbxjBXK>KtzAC%Iau7<17Emr+13voWKvE#m5&-G*ti3RDO$KjEuYx_AuZd!rDoXu+&da^24)<4Y<}{&d(s<@H#+b3B!-@}MvZh$@ z>pKxD38Qm{^!SIK{|CAbBUoK#4KX!Z(2=Z4y6%2JUfvmW{Is3OVlH0F`i$2WDL`t0 z3c2x!;NLsm+&3qh2CD4DWgM>`B6Jp~UCRQ^Y9lN^IR-uHgM_Jn3JL0K;q!VS!M25$ zXtilEbdD{e_UXq^tw~(4eh;SuFZ)P-&W%71j+vFYhERU@E{6E6L;okA$)v+9Id$_E zxsds=+M%u*!fY;+~t4`Da*mdFPL1P5dx(O;vkZ3N+my>fv%)mcwpWSa&UAf zXkHzruV%aiGskFjJXcBTw-wR=qk4KzPLevGS%C#+H$d*_e6~DELNJi#4}ROGlG;y! zIF@=J>k19=(%etx!+Z6}ky-Xo?6a0yZyqH9HZi!RVmB^3DnXwrs$t)pnfUqnY@*XF zCMY+(KrOlMq+D)q?LAKqlNvhdx8*=Y7pLLsO+)O_71q@8-ZuPoJ_lNSI>;VwZ#Jzv z99Q<8!GnX7`QCp9aBs?WdWidb-yBmYk2s9iQ&Z4&{X-~D_`>Cl_TjiWv+3FMjpRd( z8I9{3B%6IFus0kyesASUayNSe_DvQ8=P7j{y|fJ?95*lrlRc~a;)R$+b+zWYuUAto zr}xx^JKshMV(^!nEG&@WVdw7;?5W?8=wrAZcr_Zl+cGandA~7fnJEjR<8z?$eJwdB zDlFK?9(vwh0!xqiW z^Vu+&t*~O>GQpyycd4d^6KSxzNJc!=L0Geu=CbnqXEQyK(M_+`)(mEfLxySCktY~v zol%Jlx0NVA=mL^y*~ukpAF*uA{R;a$68MnMNOqC8Ja)WUg6{5xdS@4c}eUnCMx24rls!DL&F@Z4u zgsA)B%V-wpkNYyE!NxZO4w%W2!b9s}TBIm>K22WW+na$|%TmGD49VrE%lO+qL=m%n z2=AQ-G3IU}lvpjHvDzl|<;)xKc3Lhb9|BtN=L7k}Y-V4qQU$kPt*93?hi&z(#clql z;J~IBoH$Yl`UR0Rs(KNc9!`XN;!|;!gDWg4xk~3uZY5hEEruQ$VMs2Cz#E0JP;l1; zw2BRpgsNc2lgqGswJn~Qn-5A(A>bDD1JvV7==^;Sk8i=?-&1_e~um8##|2-{%L4XRqPv ztE)IpMGSZIuHs~mo%|m`O8hTBKfu0)!=RcMjE@pj1d}xlNvE3|jOX&He~$K(Iqni1 z*Ts4G8llj*e<$r&e2bP_mclk&B=L7C2I>e2%1qBf|Dl<1>$D{frv za{^|KpMh^5+#*+#O6i@OBXmKU3n+7X`-?~Su)8;v=fC?bQ`P9qXt0|Y(~@e$_PwOL z{46Q>N3d5d1K-x4KzWNG_LxBqv7Ks;DZlQLf}${Td#VN2I9t(!rIpm^*b!WKmt$s_ zgJj$3y_~N&7Y;J(F%g!L_ng0^VG%)6mo6gDN=`!j_sN{sr3RaZ^2vWH@2Njrh251b zXobF~5jQ>2$i|kW#yXNilRnZTQ7>_yLMCA?X5+@1Kr&q~iIsFHp;H3Qh%9&QJ-)q! z1)CPb3iYq-Th*g@xrNJ47S~gq?KkPu<| z{`>ccntW8@J{(*hTFyJ#)jtDvh*YrDnbS(^|HHwOSdJOFMBge2P`$4Jy)S>G(UBbU z`i1K*JL*AC-unY(7c}{w-IfFP-Xgb@HR<;FY2d%*0THw+3i|tjZ?^%jVglIuZH2AgBE|vYZe-d6Y)dk0NNg3#hBVFf<)pDvM)c9 z6c4mu(cu#q#H5hXlstTCn+UUAqqz=jYuu@?Mw)&)<89i{EUmC&RF2#wr#anw_5E;E z=#l0#v(JIwng-_Eup_RR|B`Mj5#iUi$KsG&3LKr_&asAv$)J@wth{@LF_3q|xb zd;pCPD=2$R16b|tSTS-ImMD6o=~DsN`j{faX}i@+T$ne-iR9c4b(mrN4|3$?1uK2z zXz>aefq3)=(B3hj`pmOts9pYy{;J%~qeB;P^V~MP{xSj2tegs_qLt{~C=22no`96B z6FtoJYt;k{Q(vh_NRC{OmqSaak=sw4WtWIQA~Xaw8s(U(rGdi9GX!adHEj9MAR4q& zmPff>b>RsL{Ekb3U|=>#Zv-Adp@$owI5~=CHyFXQk9FW>Y=fdLfl%|Ol%(!>icMJ_ zoaeWa-7Fr0f-fA4{K0@2oKS+=u?=|U@EmOEdPle0W#Y>7?eyH*O{mc*z_%xDSe@DV zaM-Pj<$qGg)6+|cw`>tgAF69kN=lpKv;)0hZF>qYd${cQTkcs$&lm4$~z|APj0GQV$BKwa!b_z%xTk<9sN>@Ig< z{y?1=%$xOtuCn_NvcHUBhWJBldV2&dB<`W$l>*LE3Wo7UMvxK}jPdK*aY*MWiF)sf zgRhE!-^j5hx?9NDt~q$Ek1~H2WG`o+&FGPv^ ziE$*5En{j;TOq&e9Vz`NL+`1T(SNN0aN_A4L}OY0gpW7DUaE|A|67Q49Q*ptNgZS6 z3Bhv}3%H|K&MK6K(ERB^aCKE2%xO2ry+exFnsgaLKDJ{Y=e@`o%tRfoZ^E_YFSF>X z14b__rcKUGyr##cRN|>5I9ukz<%BjYlIx?{=cfsNh{}W2<)!GVmCc<)r0DrsTt{Sh zFRho~34fAx`Sw$d*#69e5bb*oer-Uisd*j$##z!bTX%?#m;vbnkEqqNXJB_H{1%af&)*WkG%PM2JqVDClbpiLr|nR0gSKT$B9F7IM?$xT^za+wBPE% z!*_{zZmgD0;8taO(;l|<_Da0n_kw+07mXuT6Zs-`gaoLnU_ctzg)1Y5zioPjn_Lt z7BJ-Lcfe~~56}U2))z_sY}bXr4E96z620p1pTR_# zuK`~M8^F-hAI#@|g5#4Dz|OuOo_qYE!CrTu?%N=74!OnhU(ZARM-Lda!V6$hERM!C zg>d49D9$=x!K#;9f!c97f#sQt%);I5gcE09a}& zYCrIk_%`&?fT2t_TXHL=w})Z`_ugw;N`TjS5q_oed}_I$JCEBvrxmYkQ2IYLLGX$y za&2l2+l?_rA4_0Np7-Dv&Q}YIVlduCi?TIM7`aD=`F%7U zyRLHi@#8Paz0MSz_Q3{TdF_OA%||hJ(@ZMTFD&@kA5AMG>PYCb9Gv%J8AQEs#-(}N zsMUcYj2B$t^^F~ZP~Cl)&O}rGJC6I9d=X+E6~J2lakP{v#e)k(z}d!%>&D6jlRiyY z9vMkB%DK)Qz8F?w6 z8_2}aNhGeR0&d<6#na!`Lvd+25sc_j?UE|0{Dyl@Dnp&j?i1!HjDF*G{X zjBBD#xhRwc%)qn3&Yqc2SD5XP=mtLV7MJLBG!&aFI3t&cfeoor(xue*@BmgHStQy zI}&r<4Xh&`!5X_a@Xf}aY(5f$uh%A%oS0OU3|oaKKk35#c_x_H|C{nhc7d;FGZuw* z(u5Zo&{d#-MjsL1zL%l{Zui-YvjkTgZzK)6H&MGoTHro?5#wbQL@rnTrC&wVnBuw7 z*e5*1xO_2#Pi8Brga)T`T%S+e0uJHB5K&B+^%3_d2Ep*}M)p=$HqF-c!Hx)fdWq}x z%l&;1HcfcLtLS04LXxdDywadfA09E*sc#W}KR7BBvD{2_$k$-cpk(m*kU;J?* z(S@AY9?#Tm%fV~06=d4;3OGH#8K->ufZuhrv2oWXoF_2_heLiry3bj~x)Eg!JlBA3biP(o@?+oGU375tN;h=|1&xYAAEOl`3n zj4qd_H%!mbzLh-O=NE_XMB-s(?*d{G{0Lg>=Mtrh!RX=hl*VRl0mtfG@@nmWa5H8G z&9ddO{7VTqeo-NosLcjCkO79N3(&YO5w{I7aHC2GmP%NYH466N*g6Cr2#I_6L-)>W)TW|{_)|6tsaTLwq4}e|!AJA1y zhHIKYWJJC3`H^wZ9WM^5jo%@OW5i8X57BoFOOC|HV@_fP8OlflEB|FiI-&4(E7S}VC0W69_QXd+Ux+3x*<2;j0n~o74r832B7aj3G#M6f{_z?n&fx~Df1;N0$eT?& zKg{MAG@WEVX>1g%OkD}8Xpar6cH#QMZ;Y7QDAcHBR38y5VK$7JW2SQj#x5O)`zbf) z10u;*hj5Hk~?D-^nzlC@zt_vir}%?qZ9V-8Ti zzYIj@Z=*A11i-gBMQ3*2B|+s1(0KnLgg>mLdn^24hk*r17k#I1*$kMQbPINOMDjlW zb-*nv!b$G8$4p4_4s6*yK(_GQs8yZ>DC!%a>nINjPW^*kzni#g>JN1I{GPG?>I@pv z`VgmELf-$lK@P3d2XE;q&@N%4dUed<+%;Zvu$z z;n>LI>$!PAi=nYIn))p|0djXg&{N(l?R``Qve&c0-QYfNPEH{kw>+Pm4(GBas_Kp!^e?*;$nW~w-%C^(xT2k8w-jJ{_X`-gc;j%rU57}*A+%k-7_w09;HDHP*Z z?|t+~R41j6Tfift1jFM(skWe*Z0guZ#R@y1f6*y=&{7XSDb{m&_;0iw1M%01VG^8} zfUkSG*&j;!sCfMq84w@FG|w9P`>7>rfD*es;LaxIT2PM#~f!Kl%&;Uu$CYhEHr)!I+U))dv!&heaPyZacZ$S_`@G-Gg2#h`W~b<7q(^-tNf;-Mo+VX8c-OR@y*oCZq80VJv#y zMA@t~^kYrA>vy(#?eSvncb&u4TxZzC;vpO_ScS)@coFr0W#r`ZE_~3DPxrV5!VRH0 zFz2!nWVFsV&m1%ds{Dn-rnb`K!NHIO|dd#t=~$f}+Ed$eTKK^6ttH zI$mNsao^+(YIBOQV0Jt8dk}+5$vKdb%O&qzR=~@R7tli7p2~^tAf=mH@YThATz640 z{tH-8t>Q5Wc2^@Lb#YnPSuJ$%=j>{$mQ;*3R---(8&NAk6Z&%#fFvzLm)KIcu&IVj zcm7Kw&*;POf%B}+>lVV>I{`H$vWU&}B3v>v5~fF#5!31|q&`fB{&!Xb!vgO?Nq;7s zTP%*N=bk6=IEp#VC4?#KaNkpRT$)gYqjTocA+Ceh(<_uHs+>k6 z|M%wCWz8@n#FeH@_`r6C?uV0kPB6bU4gUFDVSL&7U?RQ+uC4t=W#u%mdHploWhF)p zg>>1xC=cxF5wlkL}Iuvlu0Yrhb6)NgwJV6aj_RUt-y`Ge3lBYZ-1lT=QP3F zrCUMS$&*fB!9wtxHFUv|WT@CCLcXbvL+zW~tQayMrhOX+r8kDipW~KbcCDPX%r>N1 zg(5s1Yfp@Ql*BxJPy&)=exPSxk6o#SRJGgzr#u@%tJ`PM=!q<9aV-45M`39HaTlzb zYD^v8J27TPI;>uZ5d0w_WeU)0oeF8OI zBWB^^`}A*GKkwPH0+9JGAyC!YjJLfSNmBm9z;l@LFH1Z&(IV7X& zS#FNbw~fG>eP_DDrBLUQCW-%-j+^cMY50G}AUtUvy`;aXioe%{IJt2AbrzxqW>QneJ4N%_4Vb z^2leZXL=WfVi$l|PYW%Zevx@Rp#@(YRw6NFQ8ZMvfDO?U{O<;EG=h2N0B}BAKi)0Q9 zV^aDaa8QyaUuP)sN4~BX)GfV+m5U6q_qaL?3(DaGKM(@ZZW9t63+y`Xl> z7L2#1uwH4gke_Fc$Lpl<-_Im`pM9DB?MZ=BqjT{0q7R0@dQDa9MR8B@dAc)v8UB(_ zLC?*`kf~Qerkr_BPOeKNAy==^neyZLX049Z(>Q!@Sm6fgeXh^OWhBA(2XC+=j?0zZ zaH4NYSv=)qhLMv0Ve_MtV4m}cnQh4B4!38+)c_x&)mq7>4cEi3@68yzP=uevuf#`Z zC7Hb6Tpm4j25hdLkF8Ue(x%xqWGrzO&JCN5n@Z|Q_DoeMoRWzN?0&eNb&1SW>%?;3 z9bl$rh_1ew;PmY*aep8IHFvlUdDk@N-PkV@c{l=%#mtdWlHgze=Z71|ig>5ia5_Wx z6B_P%mRKom1m>a^nw++Q^VhC2bGn83BZ40&mg|9H4{nex??eUTF3upL6IVm4END6`G@}zm$equOJYCWaAG`KP0z)5$}TvR`yHi38kzd>5%BIg z;riU>V{F7J)VcSO)IN{LJ-6;c`l7#>zib^C9Mi$^vrN!Lu^+l@Gf|s-Cr1s6iNo5n z5Hj-@*|{r^xnb9cYpYhXiW@$IT!;^><_3;UJ#`-JPyC?89&&iHNDN~;G(p*{pDy1K zMJD4TPHmZfAQb=He$Mjf{|(`Pd7h; zR5f$C5+8e<`fr$sC8psnuM6BcI1SvKjuVB5FwokbOx-=IFeIswF(=w&Cb!RD-0~5N ze(u8sofG+a3;YR3!;%h1O|T35P9tO<(kmJdsfA8H_T|>lJNg+IFrjHM75;EDp0d0@0rJL3i9{@33tB?zUUySJ=Viodh5^2;n^=`7a|%70(okf zP?gM`7!b0J4qj0Jd8}l9p5?mExJ=QDC=+z{?ML^OlLa>)hzL5JFM`VbEhw>K3{9t< zhSW*ZiTwqxk8E}hPXBE|$O~ucE31oxHy)F`-5aVsq|4}UxdWJdGKblnD9u~*Xc6Ke zBlxlQ7JD&23w?yrNYMj%^5Sv}Iq9Gz_-YZzCRy1**@q)IJ;n|c+IGODDc^~Sod&;8 zNDW>tD#5?2Jc!o$Af_+sK7?zlL5^k+ge1wJo%&38nSTZ!jz;6P+6zQma0fs19z~}s zbMU~wy>xH+aVq)eFlIJI)1f7GMC7d$=*KkEc-u2Hr%;KNoHB-=auTr9b0<~zN(HZ; z{cvBz6puP_9?IH1Xrq`+|0LCr4+aTnFg&05+`mg?%sW9rAkX#lbN#wg%jvq)<|x>B z8D*$9^ep~FbSx@?IhzVcxXiTly9M~xg!9JMg+h<OP2Q1FXBv0PPvN5wh6VEmlE8iA@ zN>2{eOZc+XHemGidns(l|CoG$d|@=Y9;iT?pyHz zL$@?jD?2Y%K8?$Yd}yQ(EV#a|KOW$%mJZRsX7J78a>)9Lcj3{SEKc)Kg3w?4V9N9W z5ZCGAjW5ndf$Mqd{jUluzCD7aM}=VU_zYO|HwM3**aq!%D>fgLAO|ECSO1i#WLpLA z$a!O14EP&}W`(9OXl({wU(Ucw`z2s?Q<8sf;e3A6%ZXs1K9{(hjmFt`WI(dBfVTN& z;<1)oq8eUBH`!!R^`hlOO)nnPuCAeNSxexv^IjV6Fa=|0p2NGh=MV>BA^2iF0ykUJ z@kU}2UAZU|8vQddc1tA?pEJ0H%T!-gRfMWF4B2U%%gQx1<5#y5__Or`rA|k%MOz+> zR}v!d(SU6iU!bdsAKHBCptn5_qk>o%tV}7Rmb`Le^>=`bYgT~!6Qjte%rutoyAFRI z+0FR(m_oF@F6g%^LR!yNW=jFf<+sD}UbPU-Syf9^EeZAqNP%IW2_4$uftJOR{79?0 zC=sy(O<5aIH00BeWE%+m=z|gCHqw{B?ciF2C*10KKvwBLqpBSn(8(~2C zz2_SI$xO$mWFB}77~mbq+C3`MG*56McuN;dm83kk)FP%wSBZmggHPE<9 zA1u53Y1;g`Frw#(SME&))qkhS4i6*J9ekDi-9MF{y;4Q?H=YDz=^V`Q{tAZiE$oOy zIlUtHmr-%}$`)?ggKw5!A>SU2$1lRCF>S&Weqe?$`6F@@KFySXgNKK!YxYfngew7b zy@d&!emaCd_UV(X`+KRKYd#8*BLt(gJq!jpd=OFg+B-pz0 zDS7eh6Em`JjJNN%9^Czy4Mu&7f$Y2sjxhsFbay*^SXhcLgaYu>LTTdXv>jhN*`dD_ zH(z9F2c$Sp;cptfh>LZ)Fv6{akyrjp63S%ILbZ*CO?XU4&Hb>xYA=*5@de`@^Qimm za8#LD%3FHnF%Iq&pk$^g4fv&j3m^Hym0nkxy)p#*Mb)WnrWQFa)dZqtO&mYe3VJ8% z@lyS7xYBr#I^NEpNnOGCl8A%DpbaB(eF5B?L~)PW6bw~gLo#1jqH}jL^6$UK+XoI~ zcKSb@XR`{Vl8fQ5pcr#MavfBCq1c-_8O>{xu_aLrx98qQj{$91_-dTs`o}!h!eW@p zdg@_Lmo+zQh``$mI)duk0gN`wF&FX6sh*7j24wi7o6o@n3+5;nLH7W#5otow;%b9>y7%4tH@!6rN~+7=y%2+efM+_t{RmdIbB8PbU9-#Gm-WDk6_!xVrKK@ zRGM`DCsY654WwM>kcLqsyQ5<##P88Ty={}ZS$4wcRJs+frH3-|J1&qKF(LSw;sq<- zX7F|fEyOeZ+4y?hEo!I}g2fWIaC2${q+Weag0{be$`$*F*{ZFW=U;+e;dYGQh7JLAF|KmicnHvtkoiOa9>av@4`| zQz#B}dBMuQOmJ1y!6~`v?AxA;IFNb>4jWCw--Mex#&u&awfsin*VWUuhoew#CQ2KA z97U^vUsOoO30y)Qv1Fb#h4NT(XO%Q}Pw&u9iyXR`UWL)Oi>US5f8fdQgLBJmF>c*M zI-bifpPK3c*Us&y_iwC$^5xu&1DCbT&C<=VX!RYMqkRM2@@G?fqZp7Fs6hUdJ7C%A z42#Yx!{eq<6#LLXA96j6y+0q~gC)E0kVr7{tUnSjr!;nBRW=b{k;1OtlZFlkpSg3$ zMX20=iE;VbgYva$RPn?x9=y~+W_NF;uLDKE?wSl(R^KC!f22XqAzSw1NI$4MJEG@G zJsSVc7Q42%!26nQq--4@ZC*(W-gI^lg4~STaTWBvasZ_lqltUOVw^Kh1VUPFprt|( z_1gc3sI9q;hYP~tb9y2WFy8V2hWq@C*|f! zDt$Enp##2JVvZp+0bXdRGJcQLVck}B+^YW^O~s-KRbK&@550zgw%f4%cnQ8wYog;< z?<6H$_Rnak0yYGm#k^-*(earZNw>L2gUJ?fKf!}joA^)@{))0wf0H*qqVcKc86q#X zid?li!}wkk1}fD?cW#=9p;HB9AbUT=+|VN85;5e*IWGVCC<$Y^-)hpDhL04~U{=a3 z*4EnqbH54U${UB_@U$)T!kx_OE$&B{ZR~RFxZgzG^=1%*MLGhnjJvSAA`5ukW|*I^ zgt5yKV3%7Q^!eJu=VuRSWz;*8ACXE{oN&U(FFvR)^BoRKi9zM0d^{`Lg{#xoqj{YgM`66Vl;Fvx5FB0=4jX@Zas5HcpgQ{% zwYECI@l5N;a+?a2m7WQ^-`hC&V{#na?|O`i z`yw%T;}~_?r3j~k-=l2wJ)D#zjE{^B=+u8%xZ*@8ah2Hxug2R_$I_ETW3@IZS#SgI z-%Eh_^h~_7R2uAO$Wy~@Q8-t>0Xh$SCGH9rx+G?n+nw_0`q|B-UhP;LWiR6aD>Z#^;Y6hsM}U z>Tuvb_DI&CDwp%#@nbh=a*l?B1>Z=!Q7o%<+Xas1v79?R2cL#jkUAyBva5sX&>dxs z&e;fKe{MqX*H%yqZiL(^wT!u^B>poEpdo={)OU9-qm;XmTEF^5Z!TU7cMHS>fr*~X zd|ofvaV(xb^KXHcOHNRI_ydBsSd#4H8 z#`l>x-!R0xueZa{I1N0WVg`QOu8{BC%<`PheWtD{m8yFjBe^o~$fYA3*Tt?Pc~o7Wj_+RlhV!;2XfHbmfjhpD%C~B;YiKsS_lrYmMHwpe zq>s$;F(603r%>^2i;4FSb6gf*2bRlD;Loa~Fc!l3>yB5@a|TbKE71V*zA_|LYKX4$ zP=^b3DV#H?x;%dP3^}~A1#b19#Pf{;VthK7{#$j2$rJGd-=iE;sAMLrlvWj-&~(Gk zA7+7DcO87>nw(1u?~|^8b98&(a$@~vEm%$%52A;}V8T6RI<0Oyw`=2Af6?FR=zL|c zco>AS!)wTaoG9w|-U)@uC1J-cJ)X3+5nFn1Dzl@l1I0ug=%T09C~|chURXW_zAA@7 zX1xGAONu~jVF@{YuNJ&(sz~B^1>VA|TS(+(W3ay2kH3P11&hvhGXvt^@%8L_vUrXc zTQ{r-TRx?e1LG25R$C+y8GVO;EV8lw~afiRbPhcm-BJYi889ys47TXxf-)FE|X=c$1rkuJa(#zkfg;)q`#M7jryB32>aCi3TYF2tjdR#H++uK>%4h zZ9e{ze2X=k$J4c3#;wSB93(CKh!)EEXkRG{N@wHXkFqfKaT#2bmUyyg5y#&n-$B0B zoMZ5v!SU4%AnR0%A{!;ZRG}YL+Cy>DrV*ToRqI|1PLcYQZ|+{kTN&E=lFy$K6a2+xK_Ex~JY;H){tm$_{5A zv=^fsvjT+IRe;q{4iOtW4~y4$(VpTC?(_U4&%Qg8aWVxQgVm0T@MP#zEjK7A+QKwS zcfgO&he7DbH9XSz0r%}mguiQz=({cYxI#RKG0BJz zAAy&-c}V^OBUskOfVKQ&qTZkaXAIRDpX&dxF7YRJJrTlE&I$VWS`GE}F9VX7DDC5Yie-fY-(zz<$Lg$QdDEtlf;@Fty6i*w?1jrfCQKK<4H#oS=Q3M+-i+RSaSDunaewbX4C(5) z23y|NPg)t)c zDTXYNkVk(DcO31sMZTvc%!&wyG;Y=`<&X*Mlw!&1KbElbi9d);DhCPmNN8+Pfc@DG zwD!v%x+9eHX4>yY#&;V$cs0n1*-U|PlDEhM@n)uSQvw9y6nMU!>msf?%N*QmiRmgE zaBEx+-J?CiNY1}va?cx@%en>H&FjN9@+>T4uC{-8^)aLgn}uCrM^RSVyG-$a}23WhuUU_i!E zkk*>SrhFC0bN+hV@0JRTbXcL%NA7;_zYhKfrUu>eNv%SNsndH5z zV7YV#%0%5q*N%m#dVV&2-NWUIpO%t01AtM}0?_u&Z!&#z1BAbq=T*r?p}No&m~NYZ zibwNVx4n@x=?|9|(XJjzm&?4=!oQ&`@gNvodo!F|SYxXSwvra0E2a-9eGxyYfh z4wqql8ctPvn{a+lIcP*gF%!m0G0gV|VOCtkkT>G=MCK-3pTqFFfQM_ww=pJ-{iu2W zAKA*GhGm+9X_Lzd;^7>{{GG?;$v#fOQ8G#3W>|~c{AC33g&MGSo;QTfS%?Qhvx(*F zc(l$cpc9uI!pg{dG~$CQZ`QGK zWHt(f$MaIB%R-8O7S5S623zEx)B7FfShn~s%H0pc2S01*{bXyrb5}M#yEsA8(*IXo+I1vE1{!6@UG@mae#GXI|my7l>!kIye+ zL)vsOSkVAAXN*C@%Ng!$mBVuPE67GafscJh(MNqIY*+CH?=69JZ?_xqojVafo(qGa z^Yh`f(__+dN0_%oe~cd98p^Jmos7?pAA)UV!!U2{v$tR-!dNSM=6GtAY7%Pz}~6%w=v_6tT(CCzyYQs)FnOH|d~yHT4_ygSp|i zr~!BW8-H;T%n8fFGk$WMKV21mC<%dW@<%#gRs~ifEhrS94m+a+L}S>Nbe>rQ$Id%a z@gu{SlBAC_F6)w#eWG}GN*o=LHy|68uHpV}Z>YVMOI2w$yP(y&ytIBjY`Qm5P-2t{ z38ASFq?nJ6m&frmrwKt-&qqe(dlPl-+JP~&nB*s&N4w9I&X#$AJ;@Vx29R8afLc zqr1>*;4VlQ{bIVg`IiCRh^ZOPjG;;#K3h0QpXqQ+v8=E7JGhkeo$n_B105J!`W5ut zW0;tC>mkSDH~r&n2fqA3@~WNTZ5(5%*2g}^>ggoGK;B&3A|oM4RB0koNzE)TEtyW9 zDo@^*|6+vpx-qZZ#*tkOmQ3y=hJYbwdDf0kr3w}*+%o=7GBR#U3Nx!&__Xv;w*;8jh9vK@EGFaIM@ zc}tr2wYQ5n1|I_^FBl9rT2<%Lmr=MVDp*&zP$ z=e!B_@g~X>cHr87_n24r=KvG#P5&k8VOP~!a{O!)S>ecs1OHkO^)-31(=!NluSBms z%Ycy#gp>e|U!E9B{Y{Rc`Ge&)I^@&qPv2>nKTViNFDV6pB|H za#=cg?(>F0IsJe>(n7pLpYMZiS1t~X-DBqM8m2p^YvAF#zZjW)FY)p6QF<`B1l;t^ zOrG~?;x~RYo!c9Z>wZY^3@$H4vt8B1`r0_iTQNY4w+iD;y%coRu!4?(|44k#bxc^j z8E%?HK=AQq_~avl5?oeKb-_%GwtvnXytM@+_}k#wxzi?o9>Lf@xC=Dm{2}&jA>36s zOa~_YKb>KQ_f8QPJ8Z}7Yn!OdK7#${ zSE07b7kHc#K$2U-VSR}SEUajS?X7<(Srd)&Ke?Sqa1Ja!la1eRm19dnGJR7vOdaA@ zW6pkr)J~xiL21(hclcXvw^|U zcwG6r3)<2+XYS_)nArFll}0w8mfU#3>s!aLT2+@R>-|E=f7W2wI*T+9Rg&T`E8J=A z18Y}|=QmwgfTFuy>3>3q%9~>GrI0hOsjTAqAvLgz^~R#0lMv+c5F##X;|=b2XF+8P zsGAA%re3fi>63DyOC%USF7O3aqnBvenTao&!(iP*alyv8Bydf1#Tr!sZIP)Z>Z<O`inML4SS5L2?7#`{T%{fXrk73Fs!uul&m7FhcqCy~Ud@Vt8m;a=5 zdASg(p3bh|T>X%VWH(cVz9KF=4=yV=Nla)Pc&-T z{X!1ZPh|^A$}EO3r7XCUT8_(4l@evOHJ}@Gl?i+Fnw41}0OGE5z~hW8$Buf1$NF!h z)MLc%bu+;uz@cnM?i%u;aS=hQkBqUhJgRG0VWV~<6*;&~Zw1i8ZP zxThwoR}|sb*#%U4U@^4Qh9j~(ss<8s&f7I-WZ zjR(Vn3>mU~&niDY1H~?TxSU!x{MGcq>AyqyI}{QjqCAHrD9*#?+1xYbyO8IwGXQiJ zc%aJa6FA*H5M37=Qsom9a8uxB!Twi;^mdsz&hD4O>pIF{BIg69`|m(fQVor$+>egS z?LfDE0;x#84ZqCB*oB#!Y2o)?rgMZag-K4(GhYBlmSkeiOCylc3Lr5tCNQ(L5`;@O z;VUs|f$i;|#E;)X=6sbVyR}8Jq<0siJD~(~UyP8sHy@B4{#Ilyw@;V(Zw+1w3ql8l z0`}L^JMggMCY?H^5+!a0aLk?AoD1+Z=I5o*y^AKo?;3I5@Xv?j{%RAZGmwW%_uR#S z1*`FgY8-C(bdo&V+k`6(Ea{A%XzJ3pmS%c{GJ5P5_@*zwYI6a;@tF<|q5I*z$`D!n zK@-oIE#??8M{$?B7VUAa!0$hY$erb4SZT2g_WLKGy^Alk@k%E{bThsUTY@`t>`AHL z6|yd^lUb&>j=Zpcz$OJRMxpiT^uS;;J(?Ipo(}KB;TMa+g3e>ylee?RE-ZL!-T*&S zZ_tvxOKj<$fx9kGXNwDBqV&ieVI<-^gxP{iLG-0fGwdE8h+jm6 zp=KzVSk;?Tmlx*rllTW_=?o(;jFG*oE7DqtSd=m?!$h06&g<3iw43(K+fD}!KY{58pRVkGLWHLtg@|j>tWR_}JXY$Y3$7^R zv*=W~W~GR~D@5QP_byrb+=?8HT?_Fo-gwx}1RmeK&H3mjVa&~w4WfT&6OvgJ!Lp2qs+z!MrON z5)BA?_uYUg8YaXjG!`DOTf#fc;aMdTk?>n-alj{xY=8&0tJsiGut_Y4$5x zA^T|*C9>5pnYD!+32Aszb%B_Qz9AcDhTzl_^)R7430gC<$i2N2ahkCxRXXK|?T0y1 z#^gir%lAKAcH$~6lW_vcxpk0TqYN=4Tah(e2!~IGP=?q+&tMe1_BZ5Rdwm@rY7cOo zT_?222*atf{-Vmxi^#v4jcRj!;8EZJ?d~qZZjn+__MfoefctqeG1HV|AIU@2sbZ`y z&VwfceR%zC9~KPqNb;Hrd^YtCUihqvI~={i#&IEU{xt&>t}N%e?7no2o1-^4w4xYH zg=H_^k(%7O=riLBopsHc9sA)-c}AU6Q|v zW2}mr;p&LzSUmMJR(CH4H>+EaBlM3{*dN6ju0JLcaTuprZ=^d8D&s@e3{#dGL*Qg{ zykVk;jaw+4r{RQCuXIA%1w9xsm4rQhm6Ri)pqFV3Uqxp*{1tjeZME*g1I?9iq(BG? z`ejk>${~D|HGn4St+4!x6-<2b3}U&R$!#Sch`VmVWqMaoZM|DiMUTRDtE1#kaU*HD zd=oor>Y>Or7i=YO;_i$f6h7Pqx-Y`vi2QT1a91)azpbaHDNC_3>mt-&_Q#$3r(vw! za;PUkxO1_p;6~U=lr}fSVdFZSYv_vaCg0}r9=BkUWC9$B&&Cy|P0)I$iXKTa#V6yQ z;aa&`QUWZY! z>C~`j9Ne2^2sYdqS`#PWDMcN+>wydZPt0b%sBSU3AIvpj_ecozm<#l~;to`Hl4l-j zHlf)%9iH9|ZODCDiZcbALtNMs6I&KSpHSN0_I6 zGf=A_lk=!|K>Sg{-j23}CmquSb0%Eimp>Om)j7@dauFlYjG2miTbJWH#{yuhm9f^+ z2TBwdup4X+V{eECq|N;1%XMb1Kydl2o|3|33onI z+&d!-*ZrGMuRc~}wh((@1p`D)evlO8JcI9FWXU$C5V#~X9ab16!c@QSFrhV(-W<`O zPkGYxgN6@$an30_6dpp;_9TPjnaf~ty_~(bCl)_9hp}IUnn}PyUtHH1MjSqCV~)rl zD%&vGnFo41@VR1ed$>)G^nBUowGM=u(2e7=O$xR-Mk8ggCBqdmH4 z-{yqoOQwR&k6mz^yA}s?CV?u}kujEgC!tb9*PhFu zP#i%EUp#`dL(;rLuPWT^DaN^xdDuMf4nI6>Bc3&%347zsaC$&IjaHMQ?{loFmm^EY zJ4{70`&7o@@kBPNR0rSm_~OHe)#$j<0wZ32qzaBn(B9e4s;+oT3|;bJ)}&C8q$nW3W`8U!UWVm;hot8Ol z@Mi`q{?=1JE_2bJ@d~Q+!|>}d0kvp9%)hs6Aunm=7R)NOq`oB+(0A8l!QMYT)J@(W zY;#9&TIM}E_*n{{svCh+=w+N;djZ>}HE~+LBCdOE%g$SO6&^gj!u&|Migz5h5Jh_# zsE9RzQfCR?a{E${{bdHab0uJv-~mQBM}X)_3yi(&&-m;pg5>@&(4NiZ1Do>DW!fLc z<%}*EWMlw^;CqwvJHLy=AMu;W-|8=9 z$%#L>bZR(#98d+)AEU7HWGu`Iog+A3>j&oc`FN>VMDRc@mtC#?7Z(?Af$S#ko#xt3 z^5RJhUP}u`ow%(io;S*Dym=n)J3NQU73%oe(3&U=aV{*2f1r{qL`3^_1x7L1@aT2~ zjwY=l2gU92;j#k8+xQ|~zoHDT?GwUa6>c{AN?LHzqMVFeR0L)1X_(b&fS<}c$$}UL zKi=<#I_G{I_$Cd3#(q#&qepwIePN{i60GwQ7X&q%V-~lY(pjQ{uWil~(frpOTj?*u zXc{uV%u}H1dNeAhgwr*r!a>(18UOQm49{L&!QVPMnAhU~8s9BRYOf!j{1ibAtj!qP zy<5=wM;}_Hh`{&4FO1{iZ{$b1JTP6~$@0|m(C4jB8wVwErfdiM=-D}@!}}V`Mn%yQ zVPPzMC;@x#YvIn4V>IEamr3tSdHh%}3P;MDsLQ23x^-JOZ8x3C{NeIfS68y!Or#2o z_HulfKrc8X`IT&vS(7FLhC| zY9@BcCStXy0VuT(k=cJK$5mMgr@lmD#KueT?1~imyJkB+pDN5N^=_ux9RjBNPZw1Y zi1Na#TItO#d0_TsF5n3XHnQRqv%vigW4U z-=8wu#WLt@qjcbpZbm;TTlzHR1UmPp6Z7lcFyFinSNK?hp=c2dNwZ|z1a)HnyoWB- zegVWJi-_;;B)<-Q#&-TN4PW(`hRK}cvMYQ#Pb?Ru@{I9pvkIyiU7}96=0f!!6(WHWVIK!K8(6NM$L_Z)uj*d6!R;dK1 zIpYMjbJB^>^qZLJuSZ|~(u4N*%jqQE8Tdyp5gm?Uc5ID1RB^L8u6Bm;ZtFXRA?TkfRk$GxCw`41{wKrej~h+!(^wR( zyWQ}eAfDc~*bW=cDDlR0u3*m2GRzcO0sSg^WJu^Ub0Tpy`gmW*_h|&&bcWFin#f?u zKbU)YjLuFv4|$XRGIOi?$g#o(#_E?f)p=8j4N?&(R1<=+?KN=Tj&opC{UEn3JWL|? zUm*LZ8ViIPrh)j-2*imA!+ekVm=#>lSR5&44tZ5$hJF#)>4ngz=HswBwVZ}`*+7I$ z6|1v^kE_F)@%z+&M1QvqH=CGF4um$5DZkC>jFxX?x5^4kd7nul&&YuK-j~ohIz`}G zR)~QYFW{%)WMEH}gXouQ00t&F_>0SM&j|WT zB1W@xqiC?%5^!2*i#qzN6KZb`)a zutU^6T?P{NX41Jn%J^fe7R*W5iXpn0bmBjIc+~vCMCyP0W|}3Fx13hRoD^g`)TXp06r^`rVh{^q&!sNSPDv3#GBTS?y-Qse9Ub5<^^|>Z=OO>Qx*N4^Ynwd7_SoI?`e?1jG zt9--{!dAFv&PMvzSDL4}-vSf3KK0F8varcmm18K~VSOT&f@6&(*Ncw8^eLCc#_Hc!;G6`;OkwZmhPUbvGOF-u7M{LSFLl@a-3mjHyP_sv|c*01PfKU?7aR+))!jKGh39x}GOjVL~8 zgRMgx%RY}KWe|*ZqeqF+t7>-Jg%{YujN&%;SlE|81^8RfV#4gz(7Q2&^ql2#ey^so zHxg%%>^YgF;QkFTsI&pTh9>;m*-Ss{6_Ls}QuOSeizxOriBZz4C*Qs&Qr~D7v`$-3 z=ciV{_FFnM>aaF4A{ltuDV%hQ9i_J<`l-Y8aUgn6oiIyAiIY=0%tsmO?5jr_gk16I z&vIHW_(JDuY@xZw0j_hMm=Bf)RO0qbh<)XX-3oEUVuPsapL;|EqVN4+{&X*zcc_`JNLzsAt#{y&hyq#Y>4aBPBA{G) z3(VM*LJq8sN6+X-bi?#CD0-2Lo5XTS%gY{Y-E)+@bK_yD*Hb7de2(QyW1;F(EiQZ% zjO?;P>UBW`Z533Y+U!4A^IM#}_kYN@d!xX!7hOs1x9O4&XA{`4e-vuhKcuQFP3YE1 z$Jk(A1bsd~n!4({!J;S0fR`F^y>$(>`?bPkZ0&zo8F3HE+z^DmAl$fr1XB4j*f!`x z3oTa*PMBXIz0<;R!)iGYYTJjq%9hgMi)YEzd{JI@_XQffnvaV1S!{s$J~Fvr3%ELl zk#KoGxY!_p`>#jybAG5o{w*cQ<+${#XSl+QXWxj8_d&Q4xf*sf&V(aFAt-c>#r^NH zu~eoAyKXg8R}K%8rZBcjoP+`L;O_vv@y@Kh)P!bX#M9PNQ&pJYgSsS4)F1_FP=G#Cy&h69mD zut(npO$wqhB(IR<+NNUS$1%EPPXpcBqRuuBNWrU{C6K#~2&!d@7&nQ3CKg-Q!Jp_Z z)}crT8>>ZdWqMKBkw2=Ccf^`?GEN{9zg^_ARub51x{*60bNTxEX&~{;)iOSF^t@kz+fSr0i;}K#`G;&0ktK%DasoJ~+I(1=+(4}734zq|9Jr!< zfpaFlCO`GJ;O3s+beH>oC>$&ge&KEqZlRCoXNR)?$#2D{!<%SzbU&8mFqM8+3&bn;E@4<-1amig z8kBUUF?FwzDu3jh%xT^z%QO`nV3wRN@n}7r!<6x zJmJNd99@lGcQlFQjGYuc&(bpZ%Ka8@2jA3#uu6RaJpNfg)EedK!v%U!EniRf7OLXH zMY))m+rZrG_2!s|v!GFXHiZ4V}~y#%D&FOj>gn?X`@ z1v8-G5B!y4bcVbGDf#CG4apOMHdNz6bAP;K7D#5PRHLU^DvXXQ@WyurLDGyn*bwy} zQQ;g2wQ)1iRZ$gI_Dc!uX5~YIJ;#`QdzSB^G74I6L&=eOHSp!G4D1cIVrK_Whx;FF zP~v7RiL{>&2S&&EcRYg2$=GbT_;P4sxgU$pXa(5cMt<&1`x4RET}aTf zei)ymk7Cr0dS>+}FQlVeF0D3en* z$xNB!6H?VWo-xrgg}0l3l3;Gnvi)5h-fw$kvMz2rINJq6{(u-R`ZWiiqXp*k_tD$R z<-Q*0;Az?-?EJSK>kV#VO@1wwUzX;4Dh<%BzlI!T#qn2wx8#!6=Kz zApdz09(PjX-G16dZ*9E+sar09(qkd?)IGx%`VSFSZ53>9lf;#Bd+6?$lX)?9B5*5j z25e*}3vNG5BWADX(y|-v#P?4ohQwC40H0Ed zvB0UF^d8zmX2~5wCDR``GT{-FU#^9*4{6*itB!^bsgTfwEJ}XOgp2g9? zb!H~r6vlPT4$Vf#ZXUgzE)M0^rd;3A7k;M9=G!l_ftQ8O=>AxZV^;;yFEiJ$Q98rK z=14IZ#Q&hsd#(ZVtw;J0}PZ%rL|_4Q({66#`zN9M3W$p6|43D>}{6!jrC6sHGTZ67{Ns z3`b^yb@vM3-|?k>#qMZ$*+I||J&R|x;1S*0Tmo6gQ*or>D+&4ShsVF3!LaDx_@sRv zWVN5gjEAAn@?8l=n#br#$#Wc+_bhoyco;pN^RwT%K(6w2*?XTw1sV0Ts6oUxxZV

Mo-f-s_FuNxhVp4If#kNox>*=?U&+Nj5^G8Wk#as4` z%QXDt!gAiN0P>Zask)wG=y#ptR4^eD*ZL{J`Q_#1C$*=*e-C@%O<)EJiytNHe3paO z-zRjRv?ecSVHE%RjASaT9L}!2dX70O^9m2=pJf&ooFr+@8my6-jzIaQh#`EW|#4%-fd;4y_VG}i`F-Kq|g!xX_JsEVu;+`$2H zf~O=0@$cO=*uB^d+6J0ICsGTiwa7Cf6|uP0J01I}H1BxDR^qu%kw$-20$rSl4;o{* z4*y-)Z2AjhbtIYQ%6WLP{tLO-qe|z^9)+C>0?gt1*+;F7$={hPaG8%h56Uj1UXeS- zcYR{YHfnL6&0?ar{{zh&{YxJ`ep~LHqYNeQ!clsBHNBYHg-0UKL+badoL|-j6UwvU z;kJ5uF!}&2_uow0xwGY0>sDGc=#SRPhfxAOxMy`bd~%YcGlaMtN7)n-zaogdTy=>F z*jx)9FB)mBPXJN<>;tDoV!(2cz`T+!m^eEZCJRE~_LeFlu#CmQ4+Xe%jvnr={K^eq zAHd?;72vVX7Xx~7LF~K{Yj{=`+qR8Et(teRMVXgac!Bv`wP(@P!ME% zFMynXFX)BGb>y*3CQ3fF#`-y%;l0od_LJFFdef%}LMM#lCGAeagZdThh1C(T@v0*f zyf8thoMbrmH3aHjJOMvdOVX-#8&}@cH<{@R;%L41#7U?>@VwdLx=ME-;)D} zy@|Gq1unXI6+iEGg&&VZ;k2PXES(z!ub+A16vtR>JGT|~-w)yk9H=AT%0uwN0bNXe z)_^CvO;Bk2L^g^0?pf{~MsKX-?hReSanDC+#@)#RZ(~QY*0vfl4{*NHByC)`b`{id z``HJTQ{Z0Seaf7Phsrof8tQTcce|I7boM^jmn36o+;I%LI-b|Sxs$S5{ph1}*`WGi z3wDSMp{z_E{Ss;f_lD%r%|!yHeRzP2x##p+odGThFNATwYrs}~E1HiRX8t{yj~WAi z$=`Za!cWs9-#2KHN4pKMRmB57a7+h@0@iM`OgBqCpJPEYzVsKN!boz41CVV=! zlRT7-g2Xjtq@3T!c-p?D$0DbL>X}0LUDiY@W!-TCFOYBsKC&{fpLQl(r`406pngII z8**%=Koi!{cKk|K%5k0aZ543);9|5D9R{Tmj#aom6Q5;E3I<+1Ab*q6aQu_2Sl1#0 zQAsmFO5q$mO}-EFnmiZ-=yfFkxObMysIzcVML9l3Q8+ctyBrdNju)-pU$}jGt;Z+sT z?)ido*=Y^;59s0YAsOuSxeLWq0oL`b;hwE1V0lG!>(xYNvH^)U>kKP_btTyK%OgEDQ zhk+G9H(rGFkE((YpGn+|=prasxBz7T~5cVQ3lXe z5e~jw51~v{i7gZCg|eVSXtOm4b}9vehd~}_^{&B=$~&}Z<4w4x&pCFkAHo2R)1#Wp zvALGXF(2jn;OC$*+!}9zA#28iMX>>18R0`KT|%OjU($pVlfikfJOsT}6*M=RVmBPc zGkgcMwP_?DeZ>VW-%=rBxCo}G{DK`1mFY%HFUWaxjW)-p<6U_v-X+xu%qFYVWS*8Y zRhCMHxTrof;&v0#Rzk2`G5}{SNWnO*7L=PM1ov~58SCLEOzWrJ_*g2IoLG8`bv1Rs z^&0beeX_e@{CjiGEfB}r{`11Yp%^e|;6B@?TQK2pFnPA?F`1aC%4LA}kuil-aG87u z&z^iv?(UPowkNNNl=BWaeApC{6xHCUWI6nP(T-1EDllH(lK2+GV%*-PjIuZ0>eWbyvHaO$f0?K9AuuF2SMT4SbJ6( zx{D-HCuo0Cl^yI7LunKYP^HJX~cg)Dg9)<0e*cMZgY_>mt_V&*V9teW#{3JC(Tr}OeZZ--Cj1`s4Cpr7^ypj}J>?&uGt2f8>PV1Epn+vcIXYb981 zvx4{cZsU?z4Kx$xW;S{ur2F|Nx+Gjd@T8dI)m)C@yKxzL?X2-+taO0Rde}`JT<2iy zI}31n?+U-R?<38^r%BFQcla}zz(+M{Vr;#ECj8Wac#QQ-x6r^ze(21H{$S&S~wi@ z8Lr=*DCk(e8}mD^(frk%3v03pGuciAyS-LG#(i~E;J-oLWR~-GRpWMhKVsD$3MXew zh5Z9hh_u^b5O?VSIX6wnbjW2^2Ht^;&|D%Rn@Nq$g3-5khTx!4B|Wb%g^lf-jAz?w z69vN#2uQz&GS1@YWmZAtP2AATG=^FRJHVIHS#YD?26yMZK^hYPcRr@Wl7EAsr|(Bc zB6&ExErvaqc7b2a`wLgyk70|fChXXw!A8WtVU~g!USI>D;?GGK=)Q>Q!HZCk=!QPa zpV0|D#iVOMTG0PZfKO^dAzk(RY8mBnNM%=<%WsSnKB=S*M)$> zE;HN_*9j(TLU5T~D68Eh%ZmwJg#KC0bjBhJFrOg98$KNheZ#$Uk7F?PCgn6&?hbr+ z)i0~v>_lD)PvU8&1GzLVo}@XA;~md77sv|+_{o@pmq~CpYdQPR+?<;!@4>-5arnc%bA66D1Wue6YFt4W7%wQI{cZg?sCovarR}hh zeTSZhddZ$Iaj9(Hl&Rx*pN_su^_s zW=(eBt|&XImQS*J@^7 z*Zv^EGe6+YwjM}Hn#9{+*-4K7)FV!-b;)%1I9zA*8%!mX={nN@Xjb-u6`>hu)ju6; z?z*D~*RdaWwSk`|V+!}f{GsW%qH&&*HaSqpF@vlUacOS=?c=fqf4_c418r5ole$&J ze?=5+dU+WR?;OVYof;%g>H}%_jf1v*T#rCRmGTqfSXZ$Dy3*GYTIP!gZ0fB*Nv8!r z&6lAL9LJ!iB8Ik&je{M_duYkvGNy9XcKZ6-HFOT{CtB|~Kd+0j;4t@0cV)gL3TuF9 z&zui`rFu~;EQ6}4$)V7(NqD|hgqO{%B>V^o{P$xrY}^q}6u69+LqjQewE*WkaPGMvr6;C6a{uV&#QE+>+(eFDcD zZzCGc;=I_vFe0Y=h~x&O6UTEGVS(Z$I_>);zUq{JtZ*$4E*|Jdn=32v>v~y?IVXfN zm%GRjyBoByA`qI>`CUhPaG=y=wp(C0h~=y6mg_MG zbi_!4=OnZ&Qo@W~OBwpwn%a(whdld5@XS&db)Hz_j05qw@>{;7Id(*0^#h1ob4)m49?$&Lme{9rKMy+LRLEz(jDJ z#lwj8v$mI#6Y?U1u@JNR4?`?Fklq*?Xau)MQJe^z^g-fzK$s^t?7_mPLarK+1 zTk|tIRJRXgHV|;MTn_<{&LDpFW6nOXgMqVkaO8<72rt-#S0*%(dAb?`v!QWtz~`~a z37zxcW!uI~))2tWoloJ}2)E~)?8BalN}`jkt?-uTAhDk~Nw6h;GPRtN1OKb*%%iCa zzqoJA6q%B_P)SLHM4V?|lr$Ss(LgE9G*F^a=6OyegeWSKA=7#GrII9+ib@ovq70Ry z`91G%t#?@OyMF)p>; z?7Pj{!wC3(>}6-abt83t@6C1$hGEaiJi6cU35nGd<4yj_aq3E~&3@|nkneWu89Qgr z7y9%gxMU9y-x68gYvt`QjcIfQ!_UB>6%O<{JgIY^dQG3`+mBuT{_ z!iIA3@M;$b^D~BVOIoSWiPu$ozMsY3zs4BzHv!h1OLNF|pCQ<-mx3=<)Hn{u8k(ZsL3n#RFj&e6YTcXZ$Ur_7 zb6W|s*Jb0;W#j3BNHHR_GmtS74n)_FZD?$%hgOX-)V|{$Oxp3A+COtZ`M|%->sU8d zhj)f-*q9C;+DDj+qMwN0F(*74I1dg;t${CZ<7i#3GG?jnAew!?w0md@HX4ZWcTG(L z4f%N3)GN$8DtVRol+B|mAK&9`uD>v7_?9#aohQ1=1#tXL4vb97ChZFvz-)asxt1~= zqNA3f3O5hw@?jBN&r8Q+8wDY4FC{sHvmk%M9t^hnK+-h|$iu_MC?PyXzBWqX&uVFW z*1jJ*_LWj4RUVkOx`D%^?YLrAAc?bI2-lZP;t3nqYNFZjd{@mQ*4JOODm9hIM2dil~&RTjK z8hc|I-*9JS9vYIBGXWY^E3xk$k38eN@)k>8GU?0jacrCzIC8)l9?sOj)^G_d{CW*V zJs06V)nBYu>mHPdJq!FnK`_{y0=K`(z`pYHxJl<8h}Evd1vbL4+VUPAdedlDo4_&l zLPSZ^mLeSTd{3o^ir{stAy~APlgQ*;D%q>bd$Q>O9$TWp?XPAdSE^^{shBXHEo}As zQ+{Axr-yf6PA8wfrqY>TqajITIoGTffH+@S-rz(DEb>(%>18I|EXy>Y4MP0qHlJw9 z9Dlf;PzQDz7UTShFP=`5Ty!CQ;0=2; zM-pBKQ98EY2JfcTq4ReEkd2VQ*D)*+E>J|Lj7sS8IfItp&O+dpOT=qs3lT2*z%Eqj zrkjuL!qI?Iv@Dv3=c<*-cu6k9zrGRjUz!qs*hTY`L-6*xo#?XrGsF*fg6j{CM`?8p zywg|U&K4Iiog~5^6bpe_97E~^*Go6bOfth9F*1}k3ycn@G5HCjXtFp1?EC`J?QbE> z;@MK^ox-@e><<~Mwub=;&g&R(337&==xm!dq#V=Nx%2~B>%H*3N~Xso&@a87=;dx*YQ4;8rRd{h0nfa1e=*DLlB@jZVA0X|K>! zoX+u%4h~p=()~P;h+IOqF1d);gJS5?8{s5sZ8@lLvx6Pn?_4f)VMn3~daG@Ped~9? z&kb8Jr}`yfe&1#qv$_8}S1*II@hW_hJQMPj(l95kjmf{9!=_xi28&nqv%(TVcsw!( zqskpH&Xe0KAKXDA+bLW<`>T4-tYBO^r+mqyd6G;0)&X-=*2RX_1uxEP~ZEcuK z)m|Jz>Bp*QcU2oQyVY=!QaYOb7^I4#AGnO#X8NY03*W4&AVvv1IO3jzMO>zpswIQC z^||V!4$a`VA_ZsM3k1b+t8l|#!XDjLQ9b$B3a~#YinlV_aM6KtMDkrcRHn=Wlbi36 zU)2F$ubPq$I)V3c^gBDZ)dS9Zk1!7>exO5Ap7<_RoAzr@!tpMM?j>nt|Htc)ls_LA z^Dbk!R4^_wFQu<+~rExGW9*( zenJYB_e+4(xEQc^YG$85E-*9NWx%udROE|q_9uSdZ$oTh3f-!riO%}mOgAHi?e7GM zUy(3un>i6$-Tz>EMi5>pWy!c*@i1d>1XBM=LQ}Uj753W;zs2{#=e9Ih?l>7E=d6WO za|0PxxB%v^T!jZWbM;Q4ILscEpr0p(!_;vP$wrGFoNzD*7P`ga%a3E2n{t;ZcSz&a zL)tJPD9n443%EhI1)DV*u*a+&{0FYUZI9_7DR`Orc|sBEdp{7rv0mothbAma8p35Y zKmwdg**KegIPTAbx_!^sq|P2^s#io`YL9jRhi{43re@Y{~>AQ@(#ZF?%1_Az&6^&5Z8p@Wsc2g<2^LS2Q zf-mSQ!o_A9;oCp}_6)70*saJiLk!+N^B8W;c}|S?pM`QA1}}f80M*GzB!b>?{dpaD zZ=8*PPN-pl=q1{JO_t;i^2nFIB(yA)rE*+v&b3wnWBV_n@}|pXLaQ0#9-LS$`Sc|- zbsoo~_Km=+HrJqVmn`UZ3*)LXEv|-BPdpp;p~mHz_({kZH{Ct~#kTgira73jt862a zMaM%-N+qfe3&Ml1FQ~>EC3q>q$JC`wpmmFLfDed(rCl{ScrBC8IW9=$)-IU z5oK6q9E15!E-;gfCxD_;5F~8WMuTIgv2v0Fgncc?`^+;q{kz_5>LwYK8ad7Nnl)K# z8VDgb1>x3E4AD3HKzCkAz`QR@AmQI>nBK4tT-_0$Wy;VS*T=|Y>Sd~@D9iUfr^Y#@ z17P*Re$%s7f9Zww9PekfJr1o1V0xnEc#lqgMC05@I>>pJL#MpM1qY+py-LQInwSN$ zo9$RQlY`yPpCPIth?UoQN%z?w21l)2bb=`u%GDKlETuRaHe_G z>{3x7Se@TZ=5LtI>so&RFa34F=iY*};m=I|-bsh?S10Gu;Cv#7Ec)T|+n1nvvXDI7 z-ig;Z9>>!_aSSryLJtfrQ0pU#&1uAGeFjsfbQY3M{35dB zS2K;S&*@U}ELJHY62+#MLe73il2(%mYo8q^n$sU*tiK3+_v13WirR41D1_J^@`vEJ zQC!a_g%%|`!W}Oy4y0R874io_PK$eV-Q@KwH|x>rXWm@AvAf2PPSI(DIe3t zr=Ls7FOTc+_J}V;m&B1oNl&oTtH2*eRnW!#5JYwm^x82CXDmpq?)rQKa(~pIX_5zZ z+GWq2+Ehe08Y=Kon~dq|&_D2ebpbiFw4PL$TELA96H(64f`mE-;Zc_t)cW-?xVhL3 zM1#tpf2Ta|5r0cFe8XV=@;Z$F76H-M;;3(O4726wGpP7^2-a;_gclY$6CaH>9BT1r zmq@3fw5$b7G?Zx2c@faad_gSJxV(kU4MH82Fm`7zlHKm8XCaE;42`JMj1zD}`5ZQO zw-Zr26ZY}$H2Uef3pNW$k&+v}ux819M*8$({@2e*;2&3j23udzgF9yveVPtk5)awk zsVc;!>pV<&zXqNjHK6a*hd^_h9i~TI#WPnLL4o5M2@gmS*_DbQDN;+n%wL3Yx>Yzq zd1LiwISJ-z_G$R)Bmmo8%-{!mk1jkU3~_RiOqF2(7z9Z}=x`5p8y|-aW%2ZEa~z2p zw*bG&F2s-;4G0aDC6*ghu(QgXmpP%u^wt7tTyQuMrvEI1O1(f<{$mK|tPjKffHLGA z&w#UsX7fF*jA7#jT@>vcC887jVK7&VcSHU*x|MyQYN~>~i|SA4(ZuNx7+Od^nN>43 z9_N{JW#)9@zgTQ34MD-*Ir!@PSuma~#&26?i+(Recpv1JK?k1?1p~$O-t!2I3brDL za}`jhQWY4~@|9Jps3e9;D$qa2nfmB*jM9ef zbhdaaUVbb9{>({cqRwYlAK#O5cXd<_`-?-0*J12(3JHHROcYa;LA>E7-Bdpd5;{Za za34bKJ!|~yC4?7tT|(yv835Prl6C){!DZ8{?9%6IB(pjW)qJDqp41)SXdA+D_g!#* z^E77iZzJ+l`6<~oXBJlk!m8%=f>6IJ2(nHJuxtO-ke;UkP%=%4cbD6b7nl0usjJgD zSAh-<|MrPo%Qc|Sj^`7_*}us7+zA}7XB{Ri2VCU26qTPzlDQxSO}=)lxSk9O>8(U< znR56t%aOi*n1)^pmV@Tvd(`#IH`sk~KhfBxz>Dm8Mub-b-pbNmbR>my&$isCmVbN~ zo6L;);;Yu-q+iF;qF0hIb(#eSte>MgvF zK04-w;c8ttvP=(@#d65w5m9t9AES;{K~T}K7(cYvfcp*!UX|u~I4QE38N2uyi{?dR zjfNnOI!vc%^o#SsrGSCqO}gUKRg&sq$J%MtFfDR%#A1U#ivRe5`d;!-@>@l;KT!M8=OyD1H1;nH(^Y+ths4)-3ACpR;Gkp!Mnk$04?~ReE<9hMXrw1@j>O4MK zI17d(^tI#VQ9=7Si8IiHy^o+ zrJFwxB@>RX*DVi(@%!W0wQUSD zQPIa>O~?<$chViOYGgL&Xp5#BOWmMg?M>`@{D=4n z)_@JKfz(XVft2n9$Wxc)%YzUjHWnnm z;rR5~#mjUWI#~&@CI>|TcO=L&a`6_QXKfFi`p3cz11AHaa z3YdqF*o!pBV-+50+KcNaM#J!t{pe!P&Hv>-f?Zn$z~Xs4>R9Ho%MKQipI;8cDXE#f zI&4@;r`rqOnQHn_0VV`rN@S`>F`?8=R6zvvlLKZg$L!`-EQ_T zaUE4)&9Fgj71)YDh5Ut*{3E;fLeWKkD5||~raw)AFWlV^+ak}yj1{wRyY(O%)tBHt zz7%h{#zk29eVCeCZG%s<^r(APAuUbil&==9C|hNMFMDIbsCW`|znX-9ArlP85M*XZ zlCOuj=kWyw1wCX?MlulFos}qU3#Ttd)d8@xS!Z)--&(P6kelNaNJjNXRyFhE+uc?5LbN`O=n-)31(zX1|^3 z(e%Ykq(n7HeMPt^u0}dH#*#tVoe;vOu-a-e)%|8pOdrX?xEdM$>pui>>JljZlZ2($ zrQwsqbi5`sovcfZW&#q0_=hvX;92hokt$rw6MS$Lx7JJ{fxoJ8Fy08BD!!nbylP43 z_h$sHBUudv&gXD!56rNN!Gw2F`1srl%vy2|KX@s^fLA_*cPaDhwkop5VS@Z_%Vq4Y z#HaM@gHPnCNdgWnYG&Ap&#GF^dvh+ZCQP@w2(c?3L-N=ONU7WjF@i!k_^1lU7oCG$ zdXtFDKU--3^_lAjZGnl|?@{&jdfYl@37ZmGTIE%XR~l4N{L3a>diO0^etMWY+dpPB z|GlJITd%R~mxaWuM2T+mc}R4m#*_5ZyU5X~)iAev2K>sS*k3mpmZ(s(=HPQ+*Q*V) zzNM25M{-g9F!y;xfzd zpjz|VQAsKkhOeDO+jDsK5m%CA zcl<1Z%gJ3tY>p1N73?Elu5ZTLp{wLISJ(PC{V;OLCmOA&h~DB?>G}Qpss5=JV$q(C zyPo8mdQXd{$8T!$CyiYIhbxPzn^hm!D?Ff?m9wEHb&MF+tY?Nalz3M%ljzBN)9LfA zVa(T{ThPwUbZPdfk^Wu5uvEhz=WG!J1?~*17jqH*>UGdi^>xfguWB;(^aGq~5yfA7 zyy#)&BVb#Si$gz#s@7<849#DQvB8ng%o(_kBle>t{GT06l~jPNz46@K$$RkDPe700 z`RPyIH^;)$LxxT25^Mn3C--?|zuK6u4iot(jQEV9JF2NJv>sZ`Q0-Oq~V`{PqZ zaejZXA8DxHfop8baiu{I+ibm!>Wa!ASh-nnH%B(3MN14<_)9teexCbm|21kx;p4~$tMsw z_Xt?}j05+wFj(o{OxC7bV^sEbyqfnCOK*5U`iT#+T=7|s54()> zogRpNh%xUYSe2lsV3hWudgrtj3}5L73m5JNH+}+Sif`g(7`fh9#&VR((8s2`$%v;k z*ypQ6$j5tY;QN`~w3dGjc{euGr7L|vkvrd$oR9GQR0u8e@P}K)`J^@AGi?>{0rQr( z5M0IierMhS@v$SYsBbClTJf4}j@d&ePx?srtq22|-3l0Sdl%l9R^lZmS746*4qU0X z6?g~kK%Go0T;v6UkADn3oN^ytYt4ce-282=l>pD-Tm;w8$z$?T)6Aa4nDH`Itm()+ zZ`7E~@#NojfmXjSv>9fxp+N;CV!1BJPws;s9p$Etqsb`y=PPql=_h%c9s$381mjS4 z9lJYxK2jr7sL*|n4wn1qr-kovj*mUuoDe}TOICw*;tc+moJb6ITnT~WCc%`o-=OXcVrTA;x_LG}iBmR`LS0@E{) zUfF~D?;PU3cRv^*x`r`JN`ir;)8y&tm1s6%i#HPuh{biTcj~kh!acbeov+6SkS1QjwT2UYogmGl>)#JYx^!N5Zb8z2xso9nAS{fJIs9SRFnLo`dSp zZZtweGznpbcyOud9NyeS$mZrQJ&@cfxXq8FV_-Ci|8t8F0}A9EWYIX08F!Z>J>k0oQFzi6+f7rkk` z8~=@Wqk#$UsPTOO!%N@j6@@%>T&9bKnz5L3@c~_%A50scY0$x2u3)YDoc*gY8P6%_ zQ|p_F^m(WT$Xi5FTATs#d!J)<9Rr5nHsZEv!Ql7pCbSI*Lg=1k_?5eqMh2TOb5b|c z<-%jo96Ouiuy(r6s_rWT#dQ|gpk##ISG{nu!zk^N+|Ojx8Co-{8l~kFjfn#VUaY3_qYpJ^g>t z`-9^2-SqWPKIthcTC4N>^{2z0v|O-mlj7gMsfROXS>x|2_v$k?(HPgZ7!MuSp_ek^ znDp%>DET9XD8=bP)UtD=v-Si^a6QH$yGHWxlMYRh%pn0>w$=BnKlDtT3lnQ*V&adv z{Fk;)pmi>gR8DE6G9UNjw8=R_>AuB5UXz#8v=xOZ4zh}l_+~eKkO{*VR zec)y~4roEEO%c{yuCIQO7eU{B7$)1ZzmY4p3g}VM&ZOdXblgYHwqREga?>qq-*#*hIMz-A8L09 z?+&+@&s_?NN3~Era0TL~Tt>eso)l+A(&(ofaqjwuxJ*JFZ%ME%h`Qs2O%qRuyo^@+#$cGr7uwkr{* zTG?X7VN;Y{&yCu0Y-107Mf%9g8+-OtLi29|cilzk^*_rQPwDemIrk4RI^UbfHH+c|EU`4&`~l^Cu}N*7Ji-YyiyU`a5-Z zig54#RV3o1KG`M^OtxJ&GW(e#g=G(ML%aR!9`ggNfDR7?^P_62hJ^;DcG{d7%WN2CkFWg;yYSZ#WbT1<;0+Q_M#R z8En;Rg!zv~$%)Eg8u(if2ju2Z`?W&wq%w=l#yI`4$Pae?gyCHPUdkg#cdb$u9|QsCZRF zhpu~ZpNkJ(VO!B)aX2y3y0^&tFTuu&?_5s(DqEAV84fKGL+8;V_OP8Zm)UVdIQ^Q; zzOVwdR*#aR#Xkv|%`w~zOW4YPebr+L>(F*>GwD`;L0p`yz&dy**(|pU3$(+K>_j+s zs{;QuPlF^EsJ0r6r1!S7beT>Zl~g-U(tI0m-vwf|uA4fZ zAGObbIDcP6E{({*NX2nX3pc?gNnd(O=Mfg1=4!3ENpNGRo17_PkQPNzZPks`r8gLb zLdxMFH-~-H@(7lvK0s;1erk3y93(Q{!RDR>c*LFQ9aI}hM5;R0KYda)Sv?$M7FR>` z{KXho=0Hp)43Sy?cG3y5W+1s}6WR5hV?XA+`pEU0fZu4a-;Dptjt5JhrK&l^Jgs5_%V(yBLFTNR^px zQvuXOXT#U6JXl$>1r7XM;K|@J{B<%3zrB9WsJU36X3&gaNybtAZbzmaBH z4zsPBE9lT`dCb{U4jUhh!}`Qn?yN4zubVg>Sk?+n(d58xzCqO4u{> z3FSrC)A@3ejLL#J$g@7fA$5dcE;qNoTX#C?Sds!EP08qbMS!n4Q4+dGO{vIgz2=_G@!Vv*Cd!he-dkMXbrZX{x0bF5y^r!8Vj#I{Cd?I_ zz`OQ57(=#9;At-JCF27Y^8bA9U_&2|qMKtLYAn@*x^GwMn~oB8>f~xDbE`Fb@bVAS zUXe=YxNgK*v75;FVJS#!_(+RaDL`DfD!AM|3sQ9fTn2{4-fv0xa(X^J%Z)PgM`x6Eobn{D@k^Ad=lBe$QA~M z?~t>ui(%nyL0F}?2SnQCQMzCO`C%W9pQbRNUsVqRH5U9W&JR%i&;{`NHwdnabXdNV zAUe+0#FEivbU42X*V;JaIEQf7|EeUoO%g`oSq-#hDHm1@<-zxb=A@@;Gwe7rp3Huz zL$mzuQ=y6=h%&!JjKV+Cz~|42r2KariP?!QS(A89hSzb$&87I7A4xpsi1M~?+mG8g z$I-!S64(k?@b2OdjF9SbNYX2UjlJ{u)o(=j+eh=TBcPZT>zspFfkfKo?t~xF9cLIF zB%zP)(cGG2bjPmUXn*b+j2D@TC%4JLc8>;J#qqx_I<`WVuM<7BOdY1>je~v_AIzPW zip%YuLf&|J+MnN#-W!e~pYthC4Sh;J_6U(u-*wnePmn&2Y17`j_Uj6ZM z0+jAI;{RxfK!;2v_|EnD*BlOmTUBzT@VGRjq!&U}KnOXXyccsv*21C#*67x|3RE=g z;pu=4{%}iRKKYlKDXT`p{qp-{QQSBo17$9d>Pi!6PXc_jLl;*a~8oxey-QeE{Ny=kc5# zZpGUo4&;I0I(p@LJQ>qWC%wy9xMItPIrn8@>ah^eDC5K4z9(egh#(R35y4II3qW{y z6J%^^!xyiG@Lly1YWy#OzG!#>Cn{EB;9xR-RMjGK1w2&y6oLi|E|N9n2iW)4Dp(ej zh9_;0LCI4?5E++9E}D%;yBABiJyRug_rAi3W${F~Sr(RbZ-jy1beQ=ov|8<16HZUA zXYH)AA&%pI89jHU?n{1>94>D*6d(YqU*A*JFC%z*YbY@>y$fP-xj^g^;G}^Pb&H*b z@z=#+QFjwrKemE+@7aVq_Dmy^#@ncQ=o9*7#T+>Kw-hezx5Gk_MwoE^E)`2xKr#Jm zSg32t?f;X|wN;9Wo;AW_ZA+kOqbsghybUgL?7yL)b`q}nl^Q?_7Q1y;7LQ z*J$!Ae2h zPnCyCvw9k>UrrwdUxYvV%VD-~i`m&IL-c&%!CVqez&#&6fLdu9Mt(I1^Y#=vS%c#t zMH@06)}{1v>;p1QM-xPE`5=7N!041I*n52vZ|TXK=&&`JzTC>4CAPP4EUmx9YRXew zeEJNH{LO=~oNlI|uYo+;c%LlYbd~M?wU{0%n~M92Y+&^u=RI$;1S_X}Xcr14(Po@` z%47#Da2jB}IgWPG&~yw|SHYg1H}rnuZCHE760(%nVbq&ac)qO$w0w)XVuSkRY zzX5P^Rl_ua9B5VHd`S^;_-e8foKl)Vx7aF!e(7^&>8C%KtEYypPPrKMB?(1NaOZ%< znw(>EGk!JOfio`KFgaDX=;OdVIv|?GIm>@?i6vX`coV`H4#`lN!M&(Ht`e4uFD54^ zzC-I`Tb_-;FdY_K1N#&8aH3QS^rt$(>$5zfLBz-k^AXy)c`aSAQZYBd6IZIKfHP=w+9xOBLS(^lCY-!1Kf@c zgMUmd$y+WB-s0Wp)%5TQ?>$l~L10VEtd7mrsk*9l=?#9SrZSM&pq?VBxKV`<~}PK;2O;do>#lZ8}f< zdSj~_Z_g)3!U9O-stCx~_YDuoit;B5tt8+6UZE+4X}HENiMZs(0q;`|6-}Y^Nz*QT zV>by7ffFy%UfH_v^mK3gi!=kU7lO;doXmgc~xn~9_~ z&4nE9dV?)C*XaYTI4UDjhFhOW^7YoOgs#Ls_I0E#sD>A#;Fo8F9bAgiJActx)&AUU zQXk#2SB7I9tfxL5#*Eh|T}-<5hc13l526JNJ*OohG7#Tz&9Vd{Nkl1!;U zs0r0>7{ZVw&VOa#0t&}MaZP9`j#X8|HvRYLVXg{Ji#UH#`BLZ~O9K6!N3s2(9P{h% zMi_ru2K&V(@rpN>(T4{=fn$Xn-(8@ETpbt$hX`A$TvJ0ss*S+%@B?~J(g@loL}R@0 zT(W#iE_yXfgX@-P`gVI0J*wY}otu?tOY1NgUb~KcYg6!KYzMu+EFXMVtO4JSCt!3! z8}GeQ!M9xAcgs_6*gtg*mAACS$x4%HKt?5#e(wdlbiEOsnz9uqh-wfoTh4#rn+ow4 zavnQkv@ zG;@en#Mm)y@|zdJ8Wl%?w@3)v_eM43fXft(0Vf8b z;#Ot2vx3jLvMQ*Zx)sFAJOlHqUtq20H!6}d1!inQEbG2S{4UI;>uelwZTuey^_Im| z)u}L`JIEZ`($5O5QARn3Y3!;EG0?Rlg8C&Zz=F6!A~X<8Cou`j zp&43r2nwA$4hv#pFx1VN9Jn$QJSR2bi|*y1_~{xxS`<#6#O}g7e@r0TBoxkzSHgCs zIrNwJHM;P4Cxm>^B02Zs!8~Rj+-+Gx*L-)vaT6R!U2+MNp1G5fZFjM3e=RgG)Wuxe zzx0XNK?p69q^mkMf~5YEeG}M$S56Oy^1*m{x%g9hXCR zOdvO3(h1K>o8c9c!5&Q?dxTvkkV6 z3iC4G=YXc~UR0cygM+U!uv)3GI(MBWeNPTkE%pf+N`6ObQf=`37Xf^#RX_&UPKDo6 zMfhf7I+^{?R=(qtu%8#^Co;~ z>|lkQR8gK>2UBm(xp2E0&(`k6fkr{N*O@~E+PJe!=Uy^&MurVBJVy4dScHotL*Y!z zO|tFJ3Q(FJMSY$+BkOvBTE^7Vt>+hD*^oP3&G9#;yod(gQ(2HZ_Y79o=z)_Omyvp+ z3^cM71%h{*&EFCVN5lTXdxMXV_;&?tiu*ukTvkE@JVvFiy@ng(0M>D{@z!#USSgqa z`+ZljIcL1l^IJCU%=d&|oz*ziKNQrVit(#-r+S?taA1Kvc{Jg&$95X?Q<)673ok~1>TpbIB7P`(vd$~RKa31tqumIfbc(%z4WYVSR+h#4^yxEo@HS~~7S!=+r70JOy zlWAbtxC8Fr8poemGDes52=Ht?r@~v?uhgW)n3QeEp;z1@$$5(_taRx`T#~8Fvv6nN zhrS@ljC)6>H7M{ElXQ8P%JNCYv3OkC?uygi?S_ZZcOfI@JC)Lo$3m@4vhc_|R3A~H z+5szIp?()F)m%wc%o6dOgI)EL#v-!(>q*e5b2oEUdxLM}N@>W45|rQCKu$K?rsJEm zVLrDf92i$iW&fT)|8qe!yfPXjpOrKA&yv{rxoNoKa|%`IOd};inI!PiT;8|L>tN6+ zOJ*vF3(5*wiXC(DD2ljY){pS^dnUntWiqtU| z*F9b?-g|a+E(Z=8>~iyPJG^bbtl<9u3sC73 literal 0 HcmV?d00001 diff --git a/test/input_models/references/QONNX_QuantGemm.ref.hxx b/test/input_models/references/QONNX_QuantGemm.ref.hxx new file mode 100644 index 0000000..a804db0 --- /dev/null +++ b/test/input_models/references/QONNX_QuantGemm.ref.hxx @@ -0,0 +1,1032 @@ +#include +#include + +namespace QONNX_QuantGemm_ExpectedOutput { +std::int8_t output[] = { + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, + -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, + 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, + 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, + -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, + 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, + -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, + -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, + 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, + 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, + 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, + 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, + 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, + 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, + 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, + -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, + 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, + 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, + 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, + -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, + 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, + -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, + 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, + 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, + -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, + 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, + -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, + 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, + -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, + -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, + -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, + 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, + 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, + -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, + -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, + -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, + 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, + 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, + -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, + 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, + 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, + 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, + 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, + -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, + 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, + -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, + -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, + -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, + 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, + -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, + 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, + 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, + -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, + 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, + -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, + -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, + 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, + 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, + 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, + 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, + 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, + 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, + 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, + 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, + 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, + 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, + -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, + -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, + 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, + -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, + 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, + 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, + -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, + -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, + -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, + 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, + -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, + 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, + -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, + 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, + -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, + -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, + -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, + -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, + -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, + 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, + -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, + 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, + 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, + -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, + 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, + 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, + -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, + -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, + -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, + 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, + -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, + -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, + 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, + 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, + 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, + 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, + -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, + -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, + 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, + 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, + 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, + 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, + 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, + 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, + 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, + -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, + -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, + 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, + -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, + 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, + -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, + -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, + 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, + 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, + -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, + -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, + -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, + 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, + 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, + -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, + -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, + 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, + 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, + 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, + -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, + -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, + 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, + -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, + 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, + -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, + -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, +}; +constexpr std::size_t output_size = 16384; +} diff --git a/test/input_models/references/QONNX_QuantGemm_input.ref.hxx b/test/input_models/references/QONNX_QuantGemm_input.ref.hxx new file mode 100644 index 0000000..8e37bfa --- /dev/null +++ b/test/input_models/references/QONNX_QuantGemm_input.ref.hxx @@ -0,0 +1,1035 @@ +#include +#include + +namespace QONNX_QuantGemm_Input { +// Deterministic signed-int8 carrier input for QONNX quantized GEMM. +// QONNX source graph encodes the corresponding real-domain values as +// x_f = (x_q - z_x) * s_x with s_x = 1/32 and z_x = 0. +std::int8_t input[] = { + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, + 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, + -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, + -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, + -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, + 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, + 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, + -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, + -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, + -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, + 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, + 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, + -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, + 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, + 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, + -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, +}; +constexpr std::size_t input_size = 16384; +} From 77915dc217f22af781400bdcd611907952e04d5b Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Thu, 9 Jul 2026 15:08:09 +0200 Subject: [PATCH 4/6] feat: generalized quantized GEMM and MatMul lowering --- core/CMakeLists.txt | 11 +- core/inc/SOFIE/OperatorList.hxx | 1 + core/inc/SOFIE/RModel.hxx | 13 + core/inc/SOFIE/ROperator.hxx | 10 +- core/inc/SOFIE/ROperator_QONNXQuant.hxx | 61 +- core/inc/SOFIE/ROperator_QuantizedGemm.hxx | 129 +-- core/inc/SOFIE/ROperator_QuantizedMatMul.hxx | 120 +++ core/inc/SOFIE/ROperator_QuantizedMatrix.hxx | 202 ++++ core/inc/SOFIE/RQuantization.hxx | 148 ++- core/inc/SOFIE/RQuantization_Analysis.hxx | 71 ++ core/inc/SOFIE/RQuantization_DenseLinear.hxx | 123 +++ core/inc/SOFIE/RQuantization_Parameters.hxx | 40 + core/inc/SOFIE/RQuantization_Storage.hxx | 58 + core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx | 242 ++++- core/src/RModel.cxx | 56 +- core/src/RModel_ALPAKA.cxx | 7 +- core/src/RModel_Quantization.cxx | 1007 +++++++++--------- core/src/RQuantization_Analysis.cxx | 241 +++++ core/src/RQuantization_DenseLinear.cxx | 782 ++++++++++++++ core/src/RQuantization_Parameters.cxx | 79 ++ core/src/RQuantization_Storage.cxx | 149 +++ 21 files changed, 2847 insertions(+), 703 deletions(-) create mode 100644 core/inc/SOFIE/ROperator_QuantizedMatMul.hxx create mode 100644 core/inc/SOFIE/ROperator_QuantizedMatrix.hxx create mode 100644 core/inc/SOFIE/RQuantization_Analysis.hxx create mode 100644 core/inc/SOFIE/RQuantization_DenseLinear.hxx create mode 100644 core/inc/SOFIE/RQuantization_Parameters.hxx create mode 100644 core/inc/SOFIE/RQuantization_Storage.hxx create mode 100644 core/src/RQuantization_Analysis.cxx create mode 100644 core/src/RQuantization_DenseLinear.cxx create mode 100644 core/src/RQuantization_Parameters.cxx create mode 100644 core/src/RQuantization_Storage.cxx diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index d474027..9103957 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -28,9 +28,15 @@ set(sources_headers SOFIE/ROperator_ConvTranspose.hxx SOFIE/ROperator_Gemm.hxx SOFIE/ROperator_QuantizedGemm.hxx + SOFIE/ROperator_QuantizedMatMul.hxx + SOFIE/ROperator_QuantizedMatrix.hxx SOFIE/SOFIE_Quantized.hxx SOFIE/SOFIE_QuantizedAlpaka.hxx SOFIE/RQuantization.hxx + SOFIE/RQuantization_Analysis.hxx + SOFIE/RQuantization_DenseLinear.hxx + SOFIE/RQuantization_Parameters.hxx + SOFIE/RQuantization_Storage.hxx SOFIE/ROperator_Relu.hxx SOFIE/ROperator_Tanh.hxx SOFIE/ROperator_LeakyRelu.hxx @@ -84,6 +90,10 @@ set(sources_cxx src/RModel_Base.cxx src/RModel.cxx src/RModel_Quantization.cxx + src/RQuantization_Analysis.cxx + src/RQuantization_DenseLinear.cxx + src/RQuantization_Parameters.cxx + src/RQuantization_Storage.cxx src/RModelProfiler.cxx src/RModelProfilerGPU.cxx src/RModel_ALPAKA.cxx @@ -126,4 +136,3 @@ install(TARGETS SOFIE_core install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/inc/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ) - diff --git a/core/inc/SOFIE/OperatorList.hxx b/core/inc/SOFIE/OperatorList.hxx index 03b9185..0be8b6b 100644 --- a/core/inc/SOFIE/OperatorList.hxx +++ b/core/inc/SOFIE/OperatorList.hxx @@ -1,6 +1,7 @@ #include "SOFIE/ROperator_Transpose.hxx" #include "SOFIE/ROperator_Gemm.hxx" #include "SOFIE/ROperator_QuantizedGemm.hxx" +#include "SOFIE/ROperator_QuantizedMatMul.hxx" #include "SOFIE/ROperator_Relu.hxx" #include "SOFIE/ROperator_Tanh.hxx" #include "SOFIE/ROperator_LeakyRelu.hxx" diff --git a/core/inc/SOFIE/RModel.hxx b/core/inc/SOFIE/RModel.hxx index 897277a..2a7d7c5 100644 --- a/core/inc/SOFIE/RModel.hxx +++ b/core/inc/SOFIE/RModel.hxx @@ -82,6 +82,7 @@ private: /// handled by the ONNX parser) into a single in-place kernel sequence. void FuseGemmActivations_GPU(); void BuildLoweredOperatorView(EQuantizedBackend backend = EQuantizedBackend::CPU); + void PrepareQuantizedTensorStorage(EQuantizedBackend backend); void AddQuantizedGeneratedHeaders(EQuantizedBackend backend = EQuantizedBackend::CPU); void AddLoweredQuantizedOperators(EQuantizedBackend backend = EQuantizedBackend::CPU); @@ -166,6 +167,18 @@ public: AddInitializedTensor(tensor_name, GetTemplatedType(T()), shape, data); } + template + void AddInitializedTensor(const std::string & tensor_name, const std::vector & shape, const std::vector &values) + { + size_t size = ConvertShapeToLength(shape); + if (values.size() != size) { + throw std::runtime_error("sofie: initialized tensor " + tensor_name + " data length does not match shape"); + } + std::shared_ptr data(malloc(size * sizeof(T)), free); + std::copy(values.begin(), values.end(), static_cast(data.get())); + AddInitializedTensor(tensor_name, GetTemplatedType(T()), shape, data); + } + void AddShapeTensor(const std::string & name, const std::vector & shapeValues, bool scalar = false); void AddAliasTensor(const std::string & name, const std::string & origin); bool IsAliasTensor(const std::string & tensor_name) const; diff --git a/core/inc/SOFIE/ROperator.hxx b/core/inc/SOFIE/ROperator.hxx index c685780..84439d2 100644 --- a/core/inc/SOFIE/ROperator.hxx +++ b/core/inc/SOFIE/ROperator.hxx @@ -39,13 +39,15 @@ enum class OperatorKind { UNARY_ABS=23, CLIP=24, NOT=25, - QUANTIZED_GEMM=26 + QUANTIZED_GEMM=26, + QUANTIZED_MATMUL=27 }; inline const char* toString(OperatorKind kind) { switch (kind) { case OperatorKind::GEMM: return "GEMM"; case OperatorKind::QUANTIZED_GEMM: return "QUANTIZED_GEMM"; + case OperatorKind::QUANTIZED_MATMUL: return "QUANTIZED_MATMUL"; case OperatorKind::LAYERNORM: return "LAYERNORM"; case OperatorKind::RELU: return "RELU"; case OperatorKind::CONSTANT: return "CONSTANT"; @@ -86,6 +88,12 @@ public: // Semantic graph-analysis hooks virtual bool IsQuantizationBoundary() const { return false; } + virtual std::string GetQuantizationSourceTensor() const + { + if (fInputTensorNames.empty()) + return {}; + return std::string(fInputTensorNames.front()); + } // Elementwise kernel fusion interface virtual bool IsElementwise() const { return false; } diff --git a/core/inc/SOFIE/ROperator_QONNXQuant.hxx b/core/inc/SOFIE/ROperator_QONNXQuant.hxx index 8c61cee..f9c504f 100644 --- a/core/inc/SOFIE/ROperator_QONNXQuant.hxx +++ b/core/inc/SOFIE/ROperator_QONNXQuant.hxx @@ -5,6 +5,7 @@ #include "SOFIE/ROperator.hxx" #include "SOFIE/SOFIE_common.hxx" #include "SOFIE/RQuantization.hxx" +#include "SOFIE/RQuantization_Parameters.hxx" #include #include @@ -34,12 +35,22 @@ private: double fScale = 1.0; std::int64_t fZeroPoint = 0; unsigned fBitWidth = 0; + bool fHasVectorParameters = false; double fQMin = 0.0; double fQMax = 0.0; - static float GetScalarFloat(RModel &model, const std::string &tensorName) + static std::vector GetFloatInitializer(RModel &model, const std::string &tensorName) { auto values = model.GetTensorData(tensorName); + if (values.empty()) { + throw std::runtime_error("SOFIE QONNX Quant expected non-empty FLOAT initializer " + tensorName); + } + return values; + } + + static float GetScalarFloat(RModel &model, const std::string &tensorName) + { + auto values = GetFloatInitializer(model, tensorName); if (values.size() != 1) { throw std::runtime_error("SOFIE QONNX Quant expected scalar FLOAT initializer " + tensorName); } @@ -72,6 +83,7 @@ public: } bool IsQuantizationBoundary() const override { return true; } + std::string GetQuantizationSourceTensor() const override { return fNX; } std::vector TypeInference(std::vector input) override { @@ -95,40 +107,38 @@ public: throw std::runtime_error("SOFIE QONNX Quant scale, zero-point, and bit-width must be initialized tensors"); } - fScale = static_cast(GetScalarFloat(model, fNScale)); - const double zeroPointFloat = static_cast(GetScalarFloat(model, fNZeroPoint)); + const auto scaleValues = GetFloatInitializer(model, fNScale); + const auto zeroPointValues = GetFloatInitializer(model, fNZeroPoint); const double bitWidthFloat = static_cast(GetScalarFloat(model, fNBitWidth)); - if (!(fScale > 0.0)) { - throw std::runtime_error("SOFIE QONNX Quant scale must be positive for tensor " + fNY); - } - if (std::round(zeroPointFloat) != zeroPointFloat) { - throw std::runtime_error("SOFIE QONNX Quant zero-point must be integral for tensor " + fNY); - } - if (std::round(bitWidthFloat) != bitWidthFloat || bitWidthFloat <= 0.0) { - throw std::runtime_error("SOFIE QONNX Quant bit-width must be a positive integer for tensor " + fNY); + if (std::round(bitWidthFloat) != bitWidthFloat || bitWidthFloat <= 0.0 || bitWidthFloat > 32.0) { + throw std::runtime_error("SOFIE QONNX Quant bit-width must be an integer in [1, 32] for tensor " + fNY); } - fZeroPoint = static_cast(zeroPointFloat); fBitWidth = static_cast(bitWidthFloat); - - QuantizationInfo info; - info.bitWidth = fBitWidth; - info.isSigned = fIsSigned; - info.narrow = fNarrow; - info.scale = fScale; - info.zeroPoint = fZeroPoint; - info.rounding = fRounding; - info.overflow = fOverflow; - info.granularity = EQuantizationGranularity::PerTensor; - info.axis = -1; + fShape = model.GetTensorShape(fNX); + QuantizationParameterSpec spec; + spec.scales.assign(scaleValues.begin(), scaleValues.end()); + spec.zeroPoints = ValidateIntegralZeroPoints(zeroPointValues, "SOFIE QONNX Quant " + fNY); + spec.bitWidth = fBitWidth; + spec.isSigned = fIsSigned; + spec.narrow = fNarrow; + spec.rounding = fRounding; + spec.overflow = fOverflow; + spec.scaleTensor = fNScale; + spec.zeroPointTensor = fNZeroPoint; + spec.tensorShape = fShape; + spec.context = "SOFIE QONNX Quant " + fNY; + auto info = MakeValidatedQuantizationInfo(spec); + fScale = info.scale; + fZeroPoint = info.zeroPoint; + fHasVectorParameters = info.granularity == EQuantizationGranularity::PerChannel; auto [qMin, qMax] = QuantizedIntegerRange(info); fQMin = static_cast(qMin); fQMax = static_cast(qMax); model.AddQuantizationInfo(fNY, std::move(info)); - fShape = model.GetTensorShape(fNX); model.AddIntermediateTensor(fNY, model.GetTensorType(fNX), fShape); model.AddNeededStdLib("cmath"); } @@ -139,6 +149,9 @@ public: if (fBitWidth == 0) { throw std::runtime_error("SOFIE QONNX Quant called to Generate without being initialized first"); } + if (fHasVectorParameters) { + throw std::runtime_error("SOFIE QONNX Quant literal code generation supports scalar parameters only; vector parameters require a fused quantized lowering"); + } const auto length = ConvertShapeToLength(fShape); const std::string scaledValue = "((static_cast(tensor_" + fNX + "[id]) / " + std::to_string(fScale) + ") + " + std::to_string(fZeroPoint) + ")"; diff --git a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx index af2c504..3b04954 100644 --- a/core/inc/SOFIE/ROperator_QuantizedGemm.hxx +++ b/core/inc/SOFIE/ROperator_QuantizedGemm.hxx @@ -4,6 +4,7 @@ #include "SOFIE/ROperator.hxx" #include "SOFIE/SOFIE_common.hxx" #include "SOFIE/RQuantization.hxx" +#include "SOFIE/ROperator_QuantizedMatrix.hxx" #include "SOFIE/SOFIE_Quantized.hxx" #include @@ -19,10 +20,7 @@ namespace SOFIE { -struct QuantizedGemmCodegenContext { - std::vector inputShape; - std::vector weightShape; - std::vector outputShape; +struct QuantizedGemmCodegenContext : QuantizedMatrixCodegenContext { float alpha = 1.0f; float beta = 1.0f; std::int64_t transA = 0; @@ -35,9 +33,7 @@ namespace INTERNAL { inline void ValidateQuantizedGemmContext(const QuantizedGemmCodegenContext &context, const std::string &pathName) { - if (context.inputShape.empty() || context.weightShape.empty() || context.outputShape.empty()) { - throw std::runtime_error("SOFIE " + pathName + " called before Gemm initialization"); - } + ValidateQuantizedMatrixContext(context, "Gemm", pathName); if (context.transA != 0 || context.transB != 1) { throw std::runtime_error("SOFIE " + pathName + " supports transA=0 and transB=1"); } @@ -47,9 +43,6 @@ inline void ValidateQuantizedGemmContext(const QuantizedGemmCodegenContext &cont if (context.activation != EActivationType::UNDEFINED && context.activation != EActivationType::RELU) { throw std::runtime_error("SOFIE " + pathName + " supports only no fused activation or fused ReLU"); } - if (context.inputShape.size() != 2 || context.weightShape.size() != 2 || context.outputShape.size() != 2) { - throw std::runtime_error("SOFIE " + pathName + " supports rank-2 Gemm only"); - } } inline bool HasQuantizedGemmBias(const QuantizedGemmRegion ®ion) @@ -57,40 +50,6 @@ inline bool HasQuantizedGemmBias(const QuantizedGemmRegion ®ion) return !region.biasSourceTensor.empty(); } -inline const char *QuantizedCudaInputCarrierName(EQuantizedCarrierMode mode) -{ - switch (mode) { - case EQuantizedCarrierMode::Int8: - return "Int8"; - case EQuantizedCarrierMode::Float: - return "Float"; - case EQuantizedCarrierMode::UInt8: - throw std::runtime_error("SOFIE quantized CUDA GEMM currently supports signed int8 input carriers only"); - default: - throw std::runtime_error("SOFIE quantized CUDA GEMM received unsupported input carrier mode"); - } -} - -inline const char *QuantizedCudaEpilogueModeName(EQuantizedOutputMode mode) -{ - switch (mode) { - case EQuantizedOutputMode::ExactFakeQuantFloat: - return "ExactFakeQuant"; - case EQuantizedOutputMode::Quantized: - return "Quantized"; - default: - throw std::runtime_error("SOFIE quantized CUDA GEMM received unsupported output mode"); - } -} - -inline const char *QuantizedCudaOutputCarrierName(const QuantizationInfo &info) -{ - if (info.bitWidth != 8) { - throw std::runtime_error("SOFIE quantized CUDA GEMM currently supports only 8-bit output carriers"); - } - return info.isSigned ? "Int8" : "UInt8"; -} - } // namespace INTERNAL inline std::string GenerateFusedQuantizedGemmCallCPUCode(std::string opName, const QuantizedGemmCodegenContext &context, @@ -296,63 +255,39 @@ inline std::string GenerateFusedQuantizedGemmCublasLtCoreLaunch(std::string opNa if (region.inputSourceTensor.empty() || region.outputTensor.empty()) { throw std::runtime_error("SOFIE fused Quantized Gemm cuBLASLt core launch is missing input/output tensors"); } + if (plan.weightScaleMode == EQuantizedParameterMode::PerOutputChannel && plan.weightScaleTensor.empty()) { + throw std::runtime_error("SOFIE fused Quantized Gemm cuBLASLt per-channel launch is missing a weight scale tensor"); + } const auto dimA = context.inputShape.size(); const auto dimB = context.weightShape.size(); - const auto m = context.inputShape[dimA - 2].GetVal(); - const auto k = context.inputShape[dimA - 1].GetVal(); - const auto n = context.weightShape[dimB - 2].GetVal(); - std::stringstream out; - out << "\n//--------- ROperator_QuantizedGemm cuBLASLt int8 GEMM core boundary " << opName << "\n"; - out << " // Optimized GPU boundary: stream-ordered cuBLASLt int8 GEMM selected by the lowering plan.\n"; - out << " {\n"; - out << " // Quantized lowering capability: " << plan.capabilityTag << "\n"; - out << " // Quantized lowering reason: " << plan.reason << "\n"; - out << " SOFIE::QuantizedGemmCudaLtParams params_quantizedGemm_" << opName << "{};\n"; - out << " params_quantizedGemm_" << opName << ".m = static_cast(" << m << ");\n"; - out << " params_quantizedGemm_" << opName << ".n = static_cast(" << n << ");\n"; - out << " params_quantizedGemm_" << opName << ".k = static_cast(" << k << ");\n"; - out << std::setprecision(17); - out << " params_quantizedGemm_" << opName << ".inputScale = " << region.inputQuant.scale << ";\n"; - out << " params_quantizedGemm_" << opName << ".weightScale = " << region.weightQuant.scale << ";\n"; - out << " params_quantizedGemm_" << opName << ".biasScale = " << (region.biasQuant ? region.biasQuant->scale : 1.0) << ";\n"; - out << " params_quantizedGemm_" << opName << ".outputScale = " << region.outputQuant.scale << ";\n"; - out << " params_quantizedGemm_" << opName << ".inputZeroPoint = " << region.inputQuant.zeroPoint << ";\n"; - out << " params_quantizedGemm_" << opName << ".weightZeroPoint = " << region.weightQuant.zeroPoint << ";\n"; - out << " params_quantizedGemm_" << opName << ".biasZeroPoint = " << (region.biasQuant ? region.biasQuant->zeroPoint : 0) << ";\n"; - out << " params_quantizedGemm_" << opName << ".outputZeroPoint = " << region.outputQuant.zeroPoint << ";\n"; - const auto inputRange = QuantizedIntegerRange(region.inputQuant); - const auto biasRange = region.biasQuant ? QuantizedIntegerRange(*region.biasQuant) : std::pair{0, 0}; - const auto outputRange = QuantizedIntegerRange(region.outputQuant); - out << " params_quantizedGemm_" << opName << ".inputQMin = static_cast(" << inputRange.first << ");\n"; - out << " params_quantizedGemm_" << opName << ".inputQMax = static_cast(" << inputRange.second << ");\n"; - out << " params_quantizedGemm_" << opName << ".biasQMin = static_cast(" << biasRange.first << ");\n"; - out << " params_quantizedGemm_" << opName << ".biasQMax = static_cast(" << biasRange.second << ");\n"; - out << " params_quantizedGemm_" << opName << ".outputQMin = static_cast(" << outputRange.first << ");\n"; - out << " params_quantizedGemm_" << opName << ".outputQMax = static_cast(" << outputRange.second << ");\n"; - out << " params_quantizedGemm_" << opName << ".hasBias = " << (INTERNAL::HasQuantizedGemmBias(region) ? "true" : "false") << ";\n"; - out << " params_quantizedGemm_" << opName << ".hasRelu = " << (context.activation == EActivationType::RELU ? "true" : "false") << ";\n"; - out << " params_quantizedGemm_" << opName << ".maxWorkspaceBytes = 32ULL * 1024ULL * 1024ULL;\n"; - const auto planEpilogueMode = INTERNAL::QuantizedCudaEpilogueModeName(plan.outputMode); - const auto planInputCarrier = INTERNAL::QuantizedCudaInputCarrierName(plan.inputCarrierMode); - out << " params_quantizedGemm_" << opName << ".epilogueMode = SOFIE::EQuantizedCudaEpilogueMode::" << planEpilogueMode << ";\n"; - out << " params_quantizedGemm_" << opName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::" << planInputCarrier << ";\n"; - out << " params_quantizedGemm_" << opName << ".outputCarrier = SOFIE::EQuantizedCudaOutputCarrier::" << INTERNAL::QuantizedCudaOutputCarrierName(region.outputQuant) << ";\n"; - out << " params_quantizedGemm_" << opName << ".weightType = SOFIE::EQuantizedCudaWeightType::" << (region.weightQuant.isSigned ? "Int8" : "UInt8") << ";\n"; - out << " SOFIE::QuantizedGemmCudaLt_Call(quantizedGemmCudaLtState_" << opName - << ", alpaka::getNativeHandle(queue)" - << ", alpaka::getPtrNative(deviceBuf_" << region.outputTensor << ")" - << ", alpaka::getPtrNative(deviceBuf_" << region.inputSourceTensor << ")" - << ", alpaka::getPtrNative(deviceBuf_" << plan.weightStorageTensor << ")"; - if (INTERNAL::HasQuantizedGemmBias(region)) { - out << ", alpaka::getPtrNative(deviceBuf_" << region.biasSourceTensor << ")"; - } else { - out << ", static_cast(nullptr)"; - } - out << ", params_quantizedGemm_" << opName << ");\n"; - out << " }\n"; - return out.str(); + INTERNAL::QuantizedCudaLtMatMulCall call; + call.boundaryName = "ROperator_QuantizedGemm cuBLASLt int8 GEMM core boundary " + opName; + call.stateName = "quantizedGemmCudaLtState_" + opName; + call.paramsName = "params_quantizedGemm_" + opName; + call.outputTensor = region.outputTensor; + call.inputTensor = region.inputSourceTensor; + call.weightStorageTensor = plan.weightStorageTensor; + call.biasTensor = region.biasSourceTensor; + call.weightScaleTensor = plan.weightScaleTensor; + call.m = context.inputShape[dimA - 2].GetVal(); + call.k = context.inputShape[dimA - 1].GetVal(); + call.n = context.weightShape[dimB - 2].GetVal(); + call.inputQuant = region.inputQuant; + call.weightQuant = region.weightQuant; + call.biasQuant = region.biasQuant; + call.outputQuant = region.outputQuant; + call.outputMode = plan.outputMode; + call.inputCarrierMode = plan.inputCarrierMode; + call.weightScaleMode = plan.weightScaleMode; + call.shapePolicy = plan.shapePolicy; + call.capabilityTag = plan.capabilityTag; + call.reason = plan.reason; + call.hasBias = INTERNAL::HasQuantizedGemmBias(region); + call.hasRelu = context.activation == EActivationType::RELU; + call.weightIsSigned = region.weightQuant.isSigned; + return INTERNAL::GenerateQuantizedCudaLtMatMulCall(call); } inline std::string GenerateFusedQuantizedGemmAlpakaFakeQuantLaunch(std::string opName, diff --git a/core/inc/SOFIE/ROperator_QuantizedMatMul.hxx b/core/inc/SOFIE/ROperator_QuantizedMatMul.hxx new file mode 100644 index 0000000..80c5a1a --- /dev/null +++ b/core/inc/SOFIE/ROperator_QuantizedMatMul.hxx @@ -0,0 +1,120 @@ +#ifndef SOFIE_ROPERATOR_QUANTIZED_MATMUL +#define SOFIE_ROPERATOR_QUANTIZED_MATMUL + +#include "SOFIE/ROperator.hxx" +#include "SOFIE/SOFIE_common.hxx" +#include "SOFIE/RQuantization.hxx" +#include "SOFIE/ROperator_QuantizedMatrix.hxx" + +#include +#include +#include +#include +#include + +namespace SOFIE { + +namespace INTERNAL { + +inline void ValidateQuantizedMatMulContext(const QuantizedMatrixCodegenContext &context, + const QuantizedMatMulRegion ®ion, + const QuantizedLoweringPlan &plan, + const std::string &pathName) +{ + ValidateQuantizedMatrixContext(context, "MatMul", pathName); + if (context.inputShape[1].GetVal() != context.weightShape[0].GetVal() || + context.inputShape[0].GetVal() != context.outputShape[0].GetVal() || + context.weightShape[1].GetVal() != context.outputShape[1].GetVal()) { + throw std::runtime_error("SOFIE " + pathName + " requires X[M,K] @ W[K,N] -> Y[M,N]"); + } + if (region.inputSourceTensor.empty() || region.outputTensor.empty()) { + throw std::runtime_error("SOFIE " + pathName + " is missing input/output tensors"); + } + ValidateQuantizedCudaLtMatrixPlan(plan, pathName); +} + +} // namespace INTERNAL + +inline std::string GenerateFusedQuantizedMatMulCublasLtLaunch(std::string opName, + const QuantizedMatrixCodegenContext &context, + const QuantizedMatMulRegion ®ion, + const QuantizedLoweringPlan &plan) +{ + INTERNAL::ValidateQuantizedMatMulContext(context, region, plan, "fused Quantized MatMul cuBLASLt launch"); + + INTERNAL::QuantizedCudaLtMatMulCall call; + call.boundaryName = "ROperator_QuantizedMatMul cuBLASLt int8 MatMul boundary " + opName; + call.stateName = "quantizedMatMulCudaLtState_" + opName; + call.paramsName = "params_quantizedMatMul_" + opName; + call.outputTensor = region.outputTensor; + call.inputTensor = region.inputSourceTensor; + call.weightStorageTensor = plan.weightStorageTensor; + call.biasTensor = region.epilogue.biasSourceTensor; + call.weightScaleTensor = plan.weightScaleTensor; + call.m = context.inputShape[0].GetVal(); + call.k = context.inputShape[1].GetVal(); + call.n = context.weightShape[1].GetVal(); + call.inputQuant = region.inputQuant; + call.weightQuant = region.weightQuant; + call.biasQuant = region.epilogue.biasQuant; + call.outputQuant = region.outputQuant; + call.outputMode = plan.outputMode; + call.inputCarrierMode = plan.inputCarrierMode; + call.weightScaleMode = plan.weightScaleMode; + call.shapePolicy = plan.shapePolicy; + call.capabilityTag = plan.capabilityTag; + call.reason = plan.reason; + call.hasBias = QuantizedEpilogueHasBias(region.epilogue.kind); + call.hasRelu = QuantizedEpilogueHasRelu(region.epilogue.kind); + call.weightIsSigned = region.weightQuant.isSigned; + return INTERNAL::GenerateQuantizedCudaLtMatMulCall(call); +} + +class ROperator_QuantizedMatMul final : public ROperator { +private: + QuantizedMatMulRegion fRegion; + QuantizedLoweringPlan fPlan; + QuantizedMatrixCodegenContext fContext; + +public: + ROperator_QuantizedMatMul(QuantizedMatMulRegion region, QuantizedLoweringPlan plan, + QuantizedMatrixCodegenContext context) + : fRegion(std::move(region)), fPlan(std::move(plan)), fContext(std::move(context)) + { + fKind = OperatorKind::QUANTIZED_MATMUL; + fName = "QuantizedMatMul"; + fInputTensorNames = { fRegion.inputSourceTensor, fRegion.weightSourceTensor }; + if (QuantizedEpilogueHasBias(fRegion.epilogue.kind)) { + fInputTensorNames.emplace_back(fRegion.epilogue.biasSourceTensor); + } + fOutputTensorNames = { fRegion.outputTensor }; + } + + std::vector GetStdLibs() override { return { "cstdint", "vector" }; } + + void Initialize(RModel &) override {} + + std::string Generate(std::string) override + { + throw std::runtime_error("SOFIE ROperator_QuantizedMatMul CPU code generation is not implemented"); + } + + std::string Generate_GPU_Kernel_ALPAKA(std::string) override { return ""; } + + std::string Generate_GPU_Kernel_Definitions_ALPAKA(std::string opName) override + { + if (!IsOptimizedQuantizedAlpakaPlainDevicePlan(fPlan)) { + throw std::runtime_error("SOFIE ROperator_QuantizedMatMul Alpaka code generation requires an optimized PlainDevice plan"); + } + return " SOFIE::QuantizedGemmCudaLtState quantizedMatMulCudaLtState_" + opName + "; // owns cuBLASLt state and CUDA temporaries\n"; + } + + std::string Generate_GPU_ALPAKA(std::string opName) override + { + return GenerateFusedQuantizedMatMulCublasLtLaunch(std::move(opName), fContext, fRegion, fPlan); + } +}; + +} // namespace SOFIE + +#endif // SOFIE_ROPERATOR_QUANTIZED_MATMUL diff --git a/core/inc/SOFIE/ROperator_QuantizedMatrix.hxx b/core/inc/SOFIE/ROperator_QuantizedMatrix.hxx new file mode 100644 index 0000000..fb4a080 --- /dev/null +++ b/core/inc/SOFIE/ROperator_QuantizedMatrix.hxx @@ -0,0 +1,202 @@ +#ifndef SOFIE_ROPERATOR_QUANTIZED_MATRIX +#define SOFIE_ROPERATOR_QUANTIZED_MATRIX + +#include "SOFIE/RQuantization.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +struct QuantizedMatrixCodegenContext { + std::vector inputShape; + std::vector weightShape; + std::vector outputShape; +}; + +namespace INTERNAL { + +inline void ValidateQuantizedMatrixContext(const QuantizedMatrixCodegenContext &context, + const std::string &operatorName, + const std::string &pathName) +{ + if (context.inputShape.empty() || context.weightShape.empty() || context.outputShape.empty()) { + throw std::runtime_error("SOFIE " + pathName + " called before " + operatorName + " initialization"); + } + if (context.inputShape.size() != 2 || context.weightShape.size() != 2 || context.outputShape.size() != 2) { + throw std::runtime_error("SOFIE " + pathName + " supports rank-2 " + operatorName + " only"); + } +} + +inline void ValidateQuantizedCudaLtMatrixPlan(const QuantizedLoweringPlan &plan, + const std::string &pathName) +{ + if (plan.backend != EQuantizedBackend::ALPAKA || plan.status != EQuantizedLoweringStatus::Optimized || + !QuantizedPlanUsesPrequantizedWeights(plan) || plan.weightLayout != EQuantizedLayout::PlainDevice || + plan.weightStorageTensor.empty()) { + throw std::runtime_error("SOFIE " + pathName + " requires an optimized Alpaka PlainDevice lowering plan"); + } + if (plan.weightStorage != EQuantizedStorageType::Int8) { + throw std::runtime_error("SOFIE " + pathName + " currently supports signed int8 weight storage only"); + } +} + +inline const char *QuantizedCudaInputCarrierName(EQuantizedCarrierMode mode, const std::string &pathName) +{ + switch (mode) { + case EQuantizedCarrierMode::Int8: + return "Int8"; + case EQuantizedCarrierMode::Float: + return "Float"; + case EQuantizedCarrierMode::UInt8: + throw std::runtime_error("SOFIE " + pathName + " currently supports signed int8 input carriers only"); + default: + throw std::runtime_error("SOFIE " + pathName + " received unsupported input carrier mode"); + } +} + +inline const char *QuantizedCudaEpilogueModeName(EQuantizedOutputMode mode, const std::string &pathName) +{ + switch (mode) { + case EQuantizedOutputMode::ExactFakeQuantFloat: + return "ExactFakeQuant"; + case EQuantizedOutputMode::Quantized: + return "Quantized"; + default: + throw std::runtime_error("SOFIE " + pathName + " received unsupported output mode"); + } +} + +inline const char *QuantizedCudaOutputCarrierName(const QuantizationInfo &info, const std::string &pathName) +{ + if (info.bitWidth != 8) { + throw std::runtime_error("SOFIE " + pathName + " currently supports only 8-bit output carriers"); + } + return info.isSigned ? "Int8" : "UInt8"; +} + +struct QuantizedCudaLtMatMulCall { + std::string boundaryName; + std::string stateName; + std::string paramsName; + std::string outputTensor; + std::string inputTensor; + std::string weightStorageTensor; + std::string biasTensor; + std::string weightScaleTensor; + std::string m; + std::string n; + std::string k; + QuantizedDenseLinearShapePolicy shapePolicy; + QuantizationInfo inputQuant; + QuantizationInfo weightQuant; + std::optional biasQuant; + QuantizationInfo outputQuant; + EQuantizedOutputMode outputMode = EQuantizedOutputMode::UNDEFINED; + EQuantizedCarrierMode inputCarrierMode = EQuantizedCarrierMode::UNDEFINED; + EQuantizedParameterMode weightScaleMode = EQuantizedParameterMode::Scalar; + std::string capabilityTag; + std::string reason; + bool hasBias = false; + bool hasRelu = false; + bool weightIsSigned = true; +}; + +inline std::string GenerateQuantizedCudaLtMatMulCall(const QuantizedCudaLtMatMulCall &call) +{ + if (call.outputTensor.empty() || call.inputTensor.empty() || call.weightStorageTensor.empty()) { + throw std::runtime_error("SOFIE " + call.boundaryName + " is missing input/output/weight-storage tensors"); + } + if (call.hasBias && (!call.biasQuant.has_value() || call.biasTensor.empty())) { + throw std::runtime_error("SOFIE " + call.boundaryName + " is missing bias tensor or quantization metadata"); + } + if (call.weightScaleMode == EQuantizedParameterMode::PerOutputChannel && call.weightScaleTensor.empty()) { + throw std::runtime_error("SOFIE " + call.boundaryName + " per-channel launch is missing a weight scale tensor"); + } + + const auto inputRange = QuantizedIntegerRange(call.inputQuant); + const auto biasRange = call.biasQuant ? QuantizedIntegerRange(*call.biasQuant) : std::pair{0, 0}; + const auto outputRange = QuantizedIntegerRange(call.outputQuant); + const bool paddedExecution = call.shapePolicy.policy == EQuantizedShapePolicy::Padded; + + std::stringstream out; + out << "\n//--------- " << call.boundaryName << "\n"; + out << " // Optimized GPU boundary: stream-ordered cuBLASLt int8 matrix multiply selected by the lowering plan.\n"; + out << " {\n"; + out << " // Quantized lowering capability: " << call.capabilityTag << "\n"; + out << " // Quantized lowering reason: " << call.reason << "\n"; + out << " SOFIE::QuantizedGemmCudaLtParams " << call.paramsName << "{};\n"; + out << " " << call.paramsName << ".logicalM = static_cast(" << call.m << ");\n"; + out << " " << call.paramsName << ".logicalN = static_cast(" << call.n << ");\n"; + out << " " << call.paramsName << ".logicalK = static_cast(" << call.k << ");\n"; + if (paddedExecution) { + out << " " << call.paramsName << ".m = " << call.shapePolicy.physicalM << ";\n"; + out << " " << call.paramsName << ".n = " << call.shapePolicy.physicalN << ";\n"; + out << " " << call.paramsName << ".k = " << call.shapePolicy.physicalK << ";\n"; + out << " " << call.paramsName << ".paddedExecution = true;\n"; + } else { + out << " " << call.paramsName << ".m = " << call.paramsName << ".logicalM;\n"; + out << " " << call.paramsName << ".n = " << call.paramsName << ".logicalN;\n"; + out << " " << call.paramsName << ".k = " << call.paramsName << ".logicalK;\n"; + } + out << std::setprecision(std::numeric_limits::max_digits10); + out << " " << call.paramsName << ".inputScale = " << call.inputQuant.scale << ";\n"; + out << " " << call.paramsName << ".weightScale = " << call.weightQuant.scale << ";\n"; + out << " " << call.paramsName << ".biasScale = " << (call.biasQuant ? call.biasQuant->scale : 1.0) << ";\n"; + out << " " << call.paramsName << ".outputScale = " << call.outputQuant.scale << ";\n"; + out << " " << call.paramsName << ".inputZeroPoint = " << call.inputQuant.zeroPoint << ";\n"; + out << " " << call.paramsName << ".weightZeroPoint = " << call.weightQuant.zeroPoint << ";\n"; + out << " " << call.paramsName << ".biasZeroPoint = " << (call.biasQuant ? call.biasQuant->zeroPoint : 0) << ";\n"; + out << " " << call.paramsName << ".outputZeroPoint = " << call.outputQuant.zeroPoint << ";\n"; + out << " " << call.paramsName << ".inputQMin = static_cast(" << inputRange.first << ");\n"; + out << " " << call.paramsName << ".inputQMax = static_cast(" << inputRange.second << ");\n"; + out << " " << call.paramsName << ".biasQMin = static_cast(" << biasRange.first << ");\n"; + out << " " << call.paramsName << ".biasQMax = static_cast(" << biasRange.second << ");\n"; + out << " " << call.paramsName << ".outputQMin = static_cast(" << outputRange.first << ");\n"; + out << " " << call.paramsName << ".outputQMax = static_cast(" << outputRange.second << ");\n"; + out << " " << call.paramsName << ".hasBias = " << (call.hasBias ? "true" : "false") << ";\n"; + out << " " << call.paramsName << ".hasRelu = " << (call.hasRelu ? "true" : "false") << ";\n"; + out << " " << call.paramsName << ".maxWorkspaceBytes = 32ULL * 1024ULL * 1024ULL;\n"; + out << " " << call.paramsName << ".epilogueMode = SOFIE::EQuantizedCudaEpilogueMode::" + << QuantizedCudaEpilogueModeName(call.outputMode, call.boundaryName) << ";\n"; + out << " " << call.paramsName << ".inputCarrier = SOFIE::EQuantizedCudaInputCarrier::" + << QuantizedCudaInputCarrierName(call.inputCarrierMode, call.boundaryName) << ";\n"; + out << " " << call.paramsName << ".outputCarrier = SOFIE::EQuantizedCudaOutputCarrier::" + << QuantizedCudaOutputCarrierName(call.outputQuant, call.boundaryName) << ";\n"; + out << " " << call.paramsName << ".weightType = SOFIE::EQuantizedCudaWeightType::" + << (call.weightIsSigned ? "Int8" : "UInt8") << ";\n"; + out << " " << call.paramsName << ".weightScaleMode = SOFIE::EQuantizedCudaScaleMode::" + << (call.weightScaleMode == EQuantizedParameterMode::PerOutputChannel ? "PerOutputChannel" : "PerTensor") << ";\n"; + + out << " SOFIE::QuantizedGemmCudaLt_Call(" << call.stateName + << ", alpaka::getNativeHandle(queue)" + << ", alpaka::getPtrNative(deviceBuf_" << call.outputTensor << ")" + << ", alpaka::getPtrNative(deviceBuf_" << call.inputTensor << ")" + << ", alpaka::getPtrNative(deviceBuf_" << call.weightStorageTensor << ")"; + if (call.hasBias) { + out << ", alpaka::getPtrNative(deviceBuf_" << call.biasTensor << ")"; + } else { + out << ", static_cast(nullptr)"; + } + if (call.weightScaleMode == EQuantizedParameterMode::PerOutputChannel) { + out << ", alpaka::getPtrNative(deviceBuf_" << call.weightScaleTensor << ")"; + } else { + out << ", static_cast(nullptr)"; + } + out << ", " << call.paramsName << ");\n"; + out << " }\n"; + return out.str(); +} + +} // namespace INTERNAL + +} // namespace SOFIE + +#endif // SOFIE_ROPERATOR_QUANTIZED_MATRIX diff --git a/core/inc/SOFIE/RQuantization.hxx b/core/inc/SOFIE/RQuantization.hxx index de4462b..ce07e2e 100644 --- a/core/inc/SOFIE/RQuantization.hxx +++ b/core/inc/SOFIE/RQuantization.hxx @@ -1,6 +1,7 @@ #ifndef SOFIE_RQUANTIZATION #define SOFIE_RQUANTIZATION +#include #include #include #include @@ -31,6 +32,8 @@ struct QuantizationInfo { bool narrow = false; double scale = 1.0; std::int64_t zeroPoint = 0; + std::string scaleTensor; + std::string zeroPointTensor; EQuantizationRoundingMode rounding = EQuantizationRoundingMode::UNDEFINED; EQuantizationOverflowMode overflow = EQuantizationOverflowMode::UNDEFINED; EQuantizationGranularity granularity = EQuantizationGranularity::PerTensor; @@ -114,7 +117,12 @@ enum class EQuantizedOutputMode { enum class EQuantizedComputeProfile { UNDEFINED = 0, GenericRecognized = 1, - SignedInt8SymmetricPerTensorRank2 = 2 + SignedInt8SymmetricPerTensorRank2 = 2, + SignedInt8PerTensorActivationPerChannelWeightRank2 = 3, + UnsignedInt8ActivationSignedInt8WeightRank2 = 4, + UnsignedInt8SymmetricRank2 = 5, + AsymmetricZeroPointRank2 = 6, + UnsupportedDenseLinearRank2 = 7 }; enum class EQuantizedLayout { @@ -122,16 +130,58 @@ enum class EQuantizedLayout { }; +enum class EQuantizedParameterMode { + UNDEFINED = 0, + Scalar = 1, + PerOutputChannel = 2 +}; + + +enum class EQuantizedEpilogueKind { + None = 0, + Bias = 1, + Relu = 2, + BiasRelu = 3 +}; + +inline bool QuantizedEpilogueHasBias(EQuantizedEpilogueKind kind) +{ + return kind == EQuantizedEpilogueKind::Bias || kind == EQuantizedEpilogueKind::BiasRelu; +} + +inline bool QuantizedEpilogueHasRelu(EQuantizedEpilogueKind kind) +{ + return kind == EQuantizedEpilogueKind::Relu || kind == EQuantizedEpilogueKind::BiasRelu; +} + +struct QuantizedEpilogue { + EQuantizedEpilogueKind kind = EQuantizedEpilogueKind::None; + std::string biasSourceTensor; + std::optional biasQuant; + std::optional addOpIndex; +}; + enum class EQuantizedShapePolicy { UNDEFINED = 0, Exact = 1, ExactTooSmall = 2, PaddedCandidate = 3, - Fallback = 4, - Unsupported = 5 + Padded = 4, + Fallback = 5, + Unsupported = 6 }; -struct QuantizedMatMulShapePolicy { +inline bool QuantizedShapePolicyUsesPadding(EQuantizedShapePolicy policy) +{ + return policy == EQuantizedShapePolicy::PaddedCandidate || policy == EQuantizedShapePolicy::Padded; +} + +inline bool QuantizedShapePolicyIsExecutable(EQuantizedShapePolicy policy) +{ + return policy == EQuantizedShapePolicy::Exact || policy == EQuantizedShapePolicy::Padded; +} + +struct QuantizedDenseLinearShapePolicy { EQuantizedShapePolicy policy = EQuantizedShapePolicy::UNDEFINED; std::size_t logicalM = 0; std::size_t logicalK = 0; @@ -147,6 +197,37 @@ struct QuantizedMatMulShapePolicy { std::string reason; }; +enum class EQuantizedMatMulShapeKind { + UNDEFINED = 0, + Unsupported = 1, + Rank2 = 2, + FlattenableProjection = 3, + TrueBatched = 4 +}; + +struct QuantizedMatMulShapeAssessment { + EQuantizedMatMulShapeKind kind = EQuantizedMatMulShapeKind::UNDEFINED; + std::size_t logicalM = 0; + std::size_t logicalK = 0; + std::size_t logicalN = 0; + std::vector flattenedInputShape; + std::vector flattenedOutputShape; + std::string reason; + std::vector unsupportedReasons; +}; + +inline bool QuantizedMatMulShapeIsRecognized(const QuantizedMatMulShapeAssessment &assessment) +{ + return assessment.kind == EQuantizedMatMulShapeKind::Rank2 || + assessment.kind == EQuantizedMatMulShapeKind::FlattenableProjection || + assessment.kind == EQuantizedMatMulShapeKind::TrueBatched; +} + +inline bool QuantizedMatMulShapeIsRank2Executable(const QuantizedMatMulShapeAssessment &assessment) +{ + return assessment.kind == EQuantizedMatMulShapeKind::Rank2; +} + struct QuantizedTensorStorage { std::string logicalTensor; std::string sourceTensor; @@ -157,9 +238,6 @@ struct QuantizedTensorStorage { std::vector shape; EQuantizedBackend residentBackend = EQuantizedBackend::UNDEFINED; - - bool isConstant = false; - bool isDeviceResident = false; }; inline std::size_t QuantizedStorageElementSize(EQuantizedStorageType type) @@ -235,10 +313,13 @@ struct QuantizedLoweringPlan { EQuantizedOutputMode outputMode = EQuantizedOutputMode::UNDEFINED; EQuantizedComputeProfile computeProfile = EQuantizedComputeProfile::UNDEFINED; std::string capabilityTag; - QuantizedMatMulShapePolicy shapePolicy; + QuantizedDenseLinearShapePolicy shapePolicy; std::string weightStorageTensor; EQuantizedLayout weightLayout = EQuantizedLayout::UNDEFINED; + EQuantizedParameterMode weightScaleMode = EQuantizedParameterMode::Scalar; + std::string weightScaleTensor; + std::string weightZeroPointTensor; std::vector consumedOperatorIndices; bool preservesQuantizationSemantics = false; @@ -280,6 +361,33 @@ inline bool IsOptimizedQuantizedAlpakaPlainDevicePlan(const QuantizedLoweringPla return plan.backend == EQuantizedBackend::ALPAKA && IsOptimizedQuantizedPlainDevicePlan(plan); } +struct QuantizedMatMulRegion { + // Quantized carrier tensors, i.e. outputs of quantization boundaries. + std::string inputTensor; + std::string weightTensor; + std::string matmulOutputTensor; + std::string outputTensor; + + // Source tensors consumed by the quantization boundaries. + std::string inputSourceTensor; + std::string weightSourceTensor; + + std::size_t inputQuantOpIndex = static_cast(-1); + std::size_t weightQuantOpIndex = static_cast(-1); + std::size_t matmulOpIndex = static_cast(-1); + std::size_t outputQuantOpIndex = static_cast(-1); + + QuantizedEpilogue epilogue; + QuantizedMatMulShapeAssessment shape; + + QuantizationInfo inputQuant; + QuantizationInfo weightQuant; + QuantizationInfo outputQuant; + + EQuantizedLoweringStatus status = EQuantizedLoweringStatus::UNDEFINED; + std::string reason; +}; + struct QuantizedGemmRegion { // Quantized carrier tensors, i.e. outputs of quantization boundaries. std::string inputTensor; @@ -318,16 +426,40 @@ struct QuantizationModelState { std::unordered_map tensorInfos; std::unordered_map tensorStorages; std::unordered_map gemmRegions; + std::unordered_map matmulRegions; std::unordered_map> loweringPlans; void ClearDerivedAnalysis() { tensorStorages.clear(); gemmRegions.clear(); + matmulRegions.clear(); loweringPlans.clear(); } }; +template +std::vector SortedQuantizedRegionOperatorIndices(const RegionMap ®ions) +{ + std::vector indices; + indices.reserve(regions.size()); + for (const auto &entry : regions) + indices.push_back(entry.first); + std::sort(indices.begin(), indices.end()); + return indices; +} + +inline const QuantizedLoweringPlan *FindQuantizedLoweringPlan(const QuantizationModelState &state, + std::size_t opIndex, + EQuantizedBackend backend) +{ + auto opIt = state.loweringPlans.find(opIndex); + if (opIt == state.loweringPlans.end()) + return nullptr; + auto backendIt = opIt->second.find(backend); + return backendIt == opIt->second.end() ? nullptr : &backendIt->second; +} + } // namespace SOFIE #endif // SOFIE_RQUANTIZATION diff --git a/core/inc/SOFIE/RQuantization_Analysis.hxx b/core/inc/SOFIE/RQuantization_Analysis.hxx new file mode 100644 index 0000000..e5be74e --- /dev/null +++ b/core/inc/SOFIE/RQuantization_Analysis.hxx @@ -0,0 +1,71 @@ +#ifndef SOFIE_RQUANTIZATION_ANALYSIS +#define SOFIE_RQUANTIZATION_ANALYSIS + +#include "SOFIE/RQuantization.hxx" +#include "SOFIE/ROperator.hxx" +#include "SOFIE/ROperator_Gemm.hxx" + +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +struct QuantizationGraphIndex { + std::unordered_map producerByTensor; + std::unordered_map> consumersByTensor; +}; + +struct QuantizedDenseLinearPatternMatch { + QuantizedGemmRegion region; + QuantizedMatMulShapeAssessment matmulShape; + std::vector reasons; + bool isMatMul = false; + bool hasInlineMatMulBias = false; +}; + +QuantizedDenseLinearPatternMatch MatchQuantizedDenseLinearPattern( + const ROperator_Gemm &gemm, std::size_t opIndex, + const std::function(const std::string &)> &tensorShape); + +QuantizationGraphIndex BuildQuantizationGraphIndex(const std::vector> &operators); + +std::optional MatchQuantizationBoundaryProducer( + const QuantizationGraphIndex &graph, const std::vector> &operators, + const std::string &tensor, const std::string &role, std::vector &reasons); + +std::optional MatchSingleTensorConsumer(const QuantizationGraphIndex &graph, + const std::string &tensor, + const std::string &role, + std::vector &reasons); + +bool IsFloatAddOperator(const ROperator &op); +void CheckQuantizationInfo(const QuantizationInfo &info, const std::string &role, + std::vector &reasons); +void CheckQuantizedGemmAttributes(const QuantizedGemmRegion ®ion, + std::vector &reasons); +void CheckQuantizedGemmRank2Shape(const std::vector &shape, + const std::string &role, + std::vector &reasons); +std::vector QuantizedGemmLoweringUnsupportedReasons(const QuantizedGemmRegion ®ion); + +std::string JoinQuantizationReasons(const std::vector &reasons); + +std::vector QuantizedGemmConsumedOperatorIndices(const QuantizedGemmRegion ®ion); +std::vector QuantizedMatMulConsumedOperatorIndices(const QuantizedMatMulRegion ®ion); + +QuantizedMatMulRegion MakeQuantizedMatMulRegionFromGemmLikeRegion(const QuantizedGemmRegion ®ion); + +bool IsDenseLinearBiasLikeShape(const std::vector &biasShape, + const std::vector &outputShape); + +QuantizationInfo MakeAccumulatorBiasQuantization(const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant); + +} // namespace SOFIE + +#endif // SOFIE_RQUANTIZATION_ANALYSIS diff --git a/core/inc/SOFIE/RQuantization_DenseLinear.hxx b/core/inc/SOFIE/RQuantization_DenseLinear.hxx new file mode 100644 index 0000000..90fffba --- /dev/null +++ b/core/inc/SOFIE/RQuantization_DenseLinear.hxx @@ -0,0 +1,123 @@ +#ifndef SOFIE_RQUANTIZATION_DENSELINEAR +#define SOFIE_RQUANTIZATION_DENSELINEAR + +#include "SOFIE/RQuantization.hxx" + +#include +#include +#include +#include +#include + +namespace SOFIE { + +struct QuantizedDenseLinearProfileAssessment { + EQuantizedComputeProfile profile = EQuantizedComputeProfile::UNDEFINED; + bool cublasLtOptimizedCandidate = false; + std::vector reasons; +}; + +struct QuantizedDenseLinearOperands { + QuantizationInfo inputQuant; + QuantizationInfo weightQuant; + QuantizationInfo outputQuant; + std::optional biasQuant; + + std::vector inputShape; + std::vector weightShape; + std::vector outputShape; + + int weightOutputChannelAxis = -1; + std::string operatorName; +}; + +struct QuantizedDenseLinearCublasLtCapability { + bool optimized = false; + EQuantizedComputeProfile profile = EQuantizedComputeProfile::GenericRecognized; + std::string tag = "recognized_not_cublaslt_optimized"; + std::string reason; + QuantizedDenseLinearShapePolicy shapePolicy; +}; + +bool IsScalarPerTensor(const QuantizationInfo &info); +bool IsPerChannelAxis(const QuantizationInfo &info, int axis); + +std::vector DenseLinearQuantizationParameterUnsupportedReasons( + const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant, + const QuantizationInfo &outputQuant, + const std::optional &biasQuant, + int expectedWeightPerChannelAxis, + const std::string &operatorName); + +std::vector QuantizeTensorToInt8(const float *data, std::size_t length, const QuantizationInfo &info); +std::vector QuantizeTensorToUInt8(const float *data, std::size_t length, const QuantizationInfo &info); + +std::vector QuantizeGemmWeightTensorToInt8(const float *data, std::size_t n, std::size_t k, + const QuantizationInfo &info, + const std::vector &perChannelScale); +std::vector QuantizeMatMulWeightTensorToInt8Transposed(const float *data, std::size_t k, std::size_t n, + const QuantizationInfo &info, + const std::vector &perChannelScale); + +std::vector QuantizeGemmWeightTensorToInt8Padded(const float *data, std::size_t n, std::size_t k, + std::size_t physicalN, std::size_t physicalK, + const QuantizationInfo &info, + const std::vector &perChannelScale); +std::vector QuantizeMatMulWeightTensorToInt8TransposedPadded(const float *data, std::size_t k, std::size_t n, + std::size_t physicalK, std::size_t physicalN, + const QuantizationInfo &info, + const std::vector &perChannelScale); + +QuantizedDenseLinearProfileAssessment AssessDenseLinearComputeProfile( + const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant, + const QuantizationInfo &outputQuant, + int expectedWeightPerChannelAxis, + const std::string &operatorName); + +QuantizedDenseLinearShapePolicy MakeCublasLtShapePolicy(std::size_t m, std::size_t k, std::size_t n); + +bool IsProfitableCublasLtPaddedDenseLinearPolicy(const QuantizedDenseLinearShapePolicy &policy); +std::string ExplainCublasLtPaddedDenseLinearProfitability(const QuantizedDenseLinearShapePolicy &policy); + +QuantizedMatMulShapeAssessment AssessQuantizedMatMulShape( + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape); + +QuantizedDenseLinearCublasLtCapability AssessCublasLtDenseLinearCapability( + const QuantizedDenseLinearOperands &operands); + +QuantizedDenseLinearCublasLtCapability +SelectExecutableDenseLinearCapability(QuantizedDenseLinearCublasLtCapability capability); + +QuantizedDenseLinearOperands MakeDenseLinearOperands(const QuantizedGemmRegion ®ion, + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape); +QuantizedDenseLinearOperands MakeDenseLinearOperands(const QuantizedMatMulRegion ®ion, + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape); + +QuantizedLoweringPlan MakeUnsupportedQuantizedMatMulPlan(const QuantizedMatMulRegion ®ion, + EQuantizedBackend backend, + std::string reason, + bool preservesSemantics); +QuantizedLoweringPlan MakeMatMulAlpakaTransposedWeightStoragePlan( + const QuantizedMatMulRegion ®ion, const std::string &weightStorageTensor, + const QuantizedDenseLinearShapePolicy &shapePolicy); +QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion ®ion, + const std::string &weightStorageTensor); +QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend, + std::string reason, + bool preservesSemantics); +QuantizedLoweringPlan MakeAlpakaFakeQuantPlan(const QuantizedGemmRegion ®ion); +QuantizedLoweringPlan MakeAlpakaCublasLtCorePlan( + const QuantizedGemmRegion ®ion, const std::string &weightStorageTensor, + const QuantizedDenseLinearCublasLtCapability &capability); + +} // namespace SOFIE + +#endif // SOFIE_RQUANTIZATION_DENSELINEAR diff --git a/core/inc/SOFIE/RQuantization_Parameters.hxx b/core/inc/SOFIE/RQuantization_Parameters.hxx new file mode 100644 index 0000000..671a78b --- /dev/null +++ b/core/inc/SOFIE/RQuantization_Parameters.hxx @@ -0,0 +1,40 @@ +#ifndef SOFIE_RQUANTIZATION_PARAMETERS +#define SOFIE_RQUANTIZATION_PARAMETERS + +#include "SOFIE/RQuantization.hxx" + +#include +#include +#include +#include + +namespace SOFIE { + +struct QuantizationParameterSpec { + std::vector scales; + std::vector zeroPoints; + unsigned bitWidth = 0; + bool isSigned = false; + bool narrow = false; + EQuantizationRoundingMode rounding = EQuantizationRoundingMode::UNDEFINED; + EQuantizationOverflowMode overflow = EQuantizationOverflowMode::UNDEFINED; + std::string scaleTensor; + std::string zeroPointTensor; + std::vector tensorShape; + std::optional explicitAxis; + std::string context; +}; + +std::vector ValidateIntegralZeroPoints(const std::vector &values, + const std::string &context); + +int InferQuantizationParameterAxis(const std::vector &tensorShape, + std::size_t parameterCount, + std::optional explicitAxis, + const std::string &context); + +QuantizationInfo MakeValidatedQuantizationInfo(const QuantizationParameterSpec &spec); + +} // namespace SOFIE + +#endif // SOFIE_RQUANTIZATION_PARAMETERS diff --git a/core/inc/SOFIE/RQuantization_Storage.hxx b/core/inc/SOFIE/RQuantization_Storage.hxx new file mode 100644 index 0000000..f3ca00f --- /dev/null +++ b/core/inc/SOFIE/RQuantization_Storage.hxx @@ -0,0 +1,58 @@ +#ifndef SOFIE_RQUANTIZATION_STORAGE +#define SOFIE_RQUANTIZATION_STORAGE + +#include "SOFIE/RQuantization.hxx" +#include "SOFIE/SOFIE_common.hxx" + +#include +#include +#include +#include +#include + +namespace SOFIE { + +ETensorType TensorTypeForQuantizedStorage(EQuantizedStorageType storage); + +QuantizedTensorStorage MakeQuantizedTensorStorage(std::string logicalTensor, + std::string sourceTensor, + std::string storageTensor, + const QuantizationInfo &quantization, + EQuantizedLayout layout, + std::vector shape, + EQuantizedBackend backend); + +std::vector PackQuantizedGemmWeightsInt8(const float *data, + std::size_t n, + std::size_t k, + std::size_t tileN, + const QuantizationInfo &info); + +std::vector PackQuantizedGemmWeightsUInt8(const float *data, + std::size_t n, + std::size_t k, + std::size_t tileN, + const QuantizationInfo &info); + +using QuantizedWeightBuffer = std::variant, std::vector>; + +struct MaterializedQuantizedWeight { + QuantizedTensorStorage storage; + QuantizedWeightBuffer buffer; +}; + +MaterializedQuantizedWeight MaterializeQuantizedGemmWeight( + const QuantizedGemmRegion ®ion, const QuantizedLoweringPlan &plan, + EQuantizedBackend backend, const float *sourceData, + const std::vector &sourceShape, + const std::vector &perChannelScales); + +MaterializedQuantizedWeight MaterializeQuantizedMatMulWeight( + const QuantizedMatMulRegion ®ion, const QuantizedLoweringPlan &plan, + EQuantizedBackend backend, const float *sourceData, + const std::vector &sourceShape, + const std::vector &perChannelScales); + +} // namespace SOFIE + +#endif // SOFIE_RQUANTIZATION_STORAGE diff --git a/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx index 5ce23ee..d377e47 100644 --- a/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx +++ b/core/inc/SOFIE/SOFIE_QuantizedAlpaka.hxx @@ -34,10 +34,19 @@ enum class EQuantizedCudaOutputCarrier { UInt8 }; +enum class EQuantizedCudaScaleMode { + PerTensor, + PerOutputChannel +}; + struct QuantizedGemmCudaLtParams { std::size_t m = 0; std::size_t n = 0; std::size_t k = 0; + std::size_t logicalM = 0; + std::size_t logicalN = 0; + std::size_t logicalK = 0; + bool paddedExecution = false; double inputScale = 1.0; double weightScale = 1.0; double biasScale = 1.0; @@ -59,6 +68,7 @@ struct QuantizedGemmCudaLtParams { EQuantizedCudaInputCarrier inputCarrier = EQuantizedCudaInputCarrier::Float; EQuantizedCudaOutputCarrier outputCarrier = EQuantizedCudaOutputCarrier::Float; EQuantizedCudaWeightType weightType = EQuantizedCudaWeightType::Int8; + EQuantizedCudaScaleMode weightScaleMode = EQuantizedCudaScaleMode::PerTensor; bool enableAutotuning = true; int autotuneIterations = 3; double accumulatorToOutputScale = 0.0; @@ -91,8 +101,26 @@ inline void QuantizedGemmCudaLt_SetProfiling(bool) {} inline bool QuantizedGemmCudaLt_ProfilingEnabled() { return false; } inline QuantizedGemmCudaLtProfile QuantizedGemmCudaLt_GetLastProfile() { return {}; } +inline void QuantizedGemmCudaPadInt8Matrix(QuantizedGemmCudaStream, const std::int8_t *, std::int8_t *, + std::size_t, std::size_t, std::size_t, std::size_t, std::int8_t = 0) +{ + throw std::runtime_error("SOFIE quantized padded CUDA input path was selected, but SOFIE_USE_CUBLASLT is not enabled"); +} + +inline void QuantizedGemmCudaUnpadInt8Matrix(QuantizedGemmCudaStream, const std::int8_t *, std::int8_t *, + std::size_t, std::size_t, std::size_t, std::size_t) +{ + throw std::runtime_error("SOFIE quantized padded CUDA output path was selected, but SOFIE_USE_CUBLASLT is not enabled"); +} + +inline void QuantizedGemmCudaUnpadUInt8Matrix(QuantizedGemmCudaStream, const std::uint8_t *, std::uint8_t *, + std::size_t, std::size_t, std::size_t, std::size_t) +{ + throw std::runtime_error("SOFIE quantized padded CUDA output path was selected, but SOFIE_USE_CUBLASLT is not enabled"); +} + inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &, QuantizedGemmCudaStream, void *, const void *, - const void *, const float *, const QuantizedGemmCudaLtParams &) + const void *, const float *, const float *, const QuantizedGemmCudaLtParams &) { throw std::runtime_error("SOFIE cuBLASLt quantized GEMM path was selected, but SOFIE_USE_CUBLASLT is not enabled"); } @@ -143,27 +171,59 @@ __global__ void QuantizedGemmCudaQuantizeInputKernel(const float *input, std::in zero, qmin, qmax)); } +__global__ void QuantizedGemmCudaPadInt8MatrixKernel(const std::int8_t *__restrict__ input, + std::int8_t *__restrict__ padded, + std::size_t logicalRows, std::size_t logicalCols, + std::size_t physicalRows, std::size_t physicalCols, + std::int8_t padValue) +{ + const std::size_t elements = physicalRows * physicalCols; + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= elements) + return; + const std::size_t row = idx / physicalCols; + const std::size_t col = idx % physicalCols; + padded[idx] = (row < logicalRows && col < logicalCols) ? input[row * logicalCols + col] : padValue; +} + +template +__global__ void QuantizedGemmCudaUnpadMatrixKernel(const T *__restrict__ padded, + T *__restrict__ output, + std::size_t logicalRows, std::size_t logicalCols, + std::size_t physicalRows, std::size_t physicalCols) +{ + const std::size_t elements = logicalRows * logicalCols; + const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (idx >= elements) + return; + const std::size_t row = idx / logicalCols; + const std::size_t col = idx % logicalCols; + output[idx] = row < physicalRows && col < physicalCols ? padded[row * physicalCols + col] : T{}; +} + __global__ void QuantizedGemmCudaBiasOutputOffsetKernel(float *__restrict__ biasOutputOffset, - const float *__restrict__ bias, - QuantizedGemmCudaLtParams params) + const float *__restrict__ bias, + const float *__restrict__ weightScaleVector, + QuantizedGemmCudaLtParams params) { const std::size_t idx = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; if (idx >= params.n) return; - int bq = __float2int_rn((bias[idx] / static_cast(params.biasScale)) + - static_cast(params.biasZeroPoint)); + const float biasScale = (params.weightScaleMode == EQuantizedCudaScaleMode::PerOutputChannel) + ? static_cast(params.inputScale * static_cast(weightScaleVector[idx])) + : static_cast(params.biasScale); + int bq = __float2int_rn((bias[idx] / biasScale) + static_cast(params.biasZeroPoint)); bq = QuantizedCudaClamp(bq, params.biasQMin, params.biasQMax); biasOutputOffset[idx] = - (static_cast(bq - params.biasZeroPoint) * static_cast(params.biasScale) / - static_cast(params.outputScale)) + + (static_cast(bq - params.biasZeroPoint) * biasScale / static_cast(params.outputScale)) + static_cast(params.outputZeroPoint); } - template __global__ void QuantizedGemmCudaQuantizedEpilogueKernel(OutputT *__restrict__ output, const std::int32_t *__restrict__ accumulator, const float *__restrict__ biasOutputOffset, + const float *__restrict__ weightScaleVector, QuantizedGemmCudaLtParams params) { const std::size_t elements = params.m * params.n; @@ -172,9 +232,12 @@ __global__ void QuantizedGemmCudaQuantizedEpilogueKernel(OutputT *__restrict__ o return; const std::size_t col = idx % params.n; + const float scale = (params.weightScaleMode == EQuantizedCudaScaleMode::PerOutputChannel) + ? static_cast((params.inputScale * static_cast(weightScaleVector[col])) / params.outputScale) + : static_cast(params.accumulatorToOutputScale); const float offset = HasBias ? biasOutputOffset[col] : static_cast(params.outputZeroPoint); int yq = __float2int_rn( - __fmaf_rn(static_cast(accumulator[idx]), static_cast(params.accumulatorToOutputScale), offset)); + __fmaf_rn(static_cast(accumulator[idx]), scale, offset)); if constexpr (HasRelu) { if (yq < params.outputZeroPoint) yq = params.outputZeroPoint; @@ -278,6 +341,52 @@ inline QuantizedGemmCudaLtProfile QuantizedGemmCudaLt_GetLastProfile() return QuantizedGemmCudaLt_LastProfileStorage(); } +inline void QuantizedGemmCudaPadInt8Matrix(QuantizedGemmCudaStream stream, const std::int8_t *input, + std::int8_t *padded, std::size_t logicalRows, + std::size_t logicalCols, std::size_t physicalRows, + std::size_t physicalCols, std::int8_t padValue = 0) +{ + const std::size_t elements = physicalRows * physicalCols; + if (elements == 0) + return; + constexpr int blockSize = 256; + const int gridSize = static_cast((elements + blockSize - 1) / blockSize); + INTERNAL::QuantizedGemmCudaPadInt8MatrixKernel<<>>( + input, padded, logicalRows, logicalCols, physicalRows, physicalCols, padValue); + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaPadInt8MatrixKernel"); +} + +template +inline void QuantizedGemmCudaUnpadMatrix(QuantizedGemmCudaStream stream, const T *padded, T *output, + std::size_t logicalRows, std::size_t logicalCols, + std::size_t physicalRows, std::size_t physicalCols) +{ + const std::size_t elements = logicalRows * logicalCols; + if (elements == 0) + return; + constexpr int blockSize = 256; + const int gridSize = static_cast((elements + blockSize - 1) / blockSize); + INTERNAL::QuantizedGemmCudaUnpadMatrixKernel<<>>( + padded, output, logicalRows, logicalCols, physicalRows, physicalCols); + INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaUnpadMatrixKernel"); +} + +inline void QuantizedGemmCudaUnpadInt8Matrix(QuantizedGemmCudaStream stream, const std::int8_t *padded, + std::int8_t *output, std::size_t logicalRows, + std::size_t logicalCols, std::size_t physicalRows, + std::size_t physicalCols) +{ + QuantizedGemmCudaUnpadMatrix(stream, padded, output, logicalRows, logicalCols, physicalRows, physicalCols); +} + +inline void QuantizedGemmCudaUnpadUInt8Matrix(QuantizedGemmCudaStream stream, const std::uint8_t *padded, + std::uint8_t *output, std::size_t logicalRows, + std::size_t logicalCols, std::size_t physicalRows, + std::size_t physicalCols) +{ + QuantizedGemmCudaUnpadMatrix(stream, padded, output, logicalRows, logicalCols, physicalRows, physicalCols); +} + struct QuantizedGemmCudaLtEventTimer { cudaEvent_t totalStart = nullptr; cudaEvent_t inputStop = nullptr; @@ -384,9 +493,11 @@ struct QuantizedGemmCudaLtState { float fSelectedCandidateMs = 0.0f; std::int8_t *fInputQuantized = nullptr; std::int32_t *fAccumulator = nullptr; + void *fOutputQuantized = nullptr; float *fBiasOutputOffset = nullptr; std::size_t fInputQuantizedBytes = 0; std::size_t fAccumulatorBytes = 0; + std::size_t fOutputQuantizedBytes = 0; std::size_t fBiasOutputOffsetBytes = 0; const float *fBiasOutputOffsetSource = nullptr; double fBiasOutputOffsetBiasScale = 0.0; @@ -396,6 +507,8 @@ struct QuantizedGemmCudaLtState { std::int32_t fBiasOutputOffsetBiasQMin = 0; std::int32_t fBiasOutputOffsetBiasQMax = 0; std::size_t fBiasOutputOffsetN = 0; + const float *fBiasOutputOffsetWeightScaleVector = nullptr; + EQuantizedCudaScaleMode fBiasOutputOffsetWeightScaleMode = EQuantizedCudaScaleMode::PerTensor; std::size_t fM = 0; std::size_t fN = 0; std::size_t fK = 0; @@ -450,6 +563,10 @@ struct QuantizedGemmCudaLtState { cudaFree(fAccumulator); fAccumulator = nullptr; } + if (fOutputQuantized != nullptr) { + cudaFree(fOutputQuantized); + fOutputQuantized = nullptr; + } if (fBiasOutputOffset != nullptr) { cudaFree(fBiasOutputOffset); fBiasOutputOffset = nullptr; @@ -469,6 +586,7 @@ struct QuantizedGemmCudaLtState { fSelectedCandidateMs = 0.0f; fInputQuantizedBytes = 0; fAccumulatorBytes = 0; + fOutputQuantizedBytes = 0; fBiasOutputOffsetBytes = 0; fBiasOutputOffsetSource = nullptr; fBiasOutputOffsetBiasScale = 0.0; @@ -478,6 +596,8 @@ struct QuantizedGemmCudaLtState { fBiasOutputOffsetBiasQMin = 0; fBiasOutputOffsetBiasQMax = 0; fBiasOutputOffsetN = 0; + fBiasOutputOffsetWeightScaleVector = nullptr; + fBiasOutputOffsetWeightScaleMode = EQuantizedCudaScaleMode::PerTensor; fM = 0; fN = 0; fK = 0; @@ -575,10 +695,26 @@ struct QuantizedGemmCudaLtState { "cudaMalloc(accumulator cache)"); fAccumulatorBytes = requiredAccumulatorBytes; } + + if (params.paddedExecution && params.epilogueMode == EQuantizedCudaEpilogueMode::Quantized) { + const std::size_t outputElementSize = params.outputCarrier == EQuantizedCudaOutputCarrier::UInt8 + ? sizeof(std::uint8_t) : sizeof(std::int8_t); + const std::size_t requiredOutputBytes = params.m * params.n * outputElementSize; + if (requiredOutputBytes > fOutputQuantizedBytes) { + if (fOutputQuantized != nullptr) { + INTERNAL::CheckCudaStatus(cudaFree(fOutputQuantized), "cudaFree(padded output cache)"); + fOutputQuantized = nullptr; + fOutputQuantizedBytes = 0; + } + INTERNAL::CheckCudaStatus(cudaMalloc(&fOutputQuantized, requiredOutputBytes), + "cudaMalloc(padded output cache)"); + fOutputQuantizedBytes = requiredOutputBytes; + } + } } const float *EnsureBiasOutputOffsetBuffer(const QuantizedGemmCudaLtParams ¶ms, const float *bias, - QuantizedGemmCudaStream stream) + const float *weightScaleVector, QuantizedGemmCudaStream stream) { if (bias == nullptr || !params.hasBias) return nullptr; @@ -602,12 +738,14 @@ struct QuantizedGemmCudaLtState { fBiasOutputOffsetBiasZeroPoint == params.biasZeroPoint && fBiasOutputOffsetOutputZeroPoint == params.outputZeroPoint && fBiasOutputOffsetBiasQMin == params.biasQMin && - fBiasOutputOffsetBiasQMax == params.biasQMax; + fBiasOutputOffsetBiasQMax == params.biasQMax && + fBiasOutputOffsetWeightScaleVector == weightScaleVector && + fBiasOutputOffsetWeightScaleMode == params.weightScaleMode; if (!cacheValid) { constexpr int threads = 256; const int blocks = static_cast((params.n + threads - 1) / threads); INTERNAL::QuantizedGemmCudaBiasOutputOffsetKernel<<>>( - fBiasOutputOffset, bias, params); + fBiasOutputOffset, bias, weightScaleVector, params); INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaBiasOutputOffsetKernel launch"); fBiasOutputOffsetSource = bias; fBiasOutputOffsetN = params.n; @@ -617,6 +755,8 @@ struct QuantizedGemmCudaLtState { fBiasOutputOffsetOutputZeroPoint = params.outputZeroPoint; fBiasOutputOffsetBiasQMin = params.biasQMin; fBiasOutputOffsetBiasQMax = params.biasQMax; + fBiasOutputOffsetWeightScaleVector = weightScaleVector; + fBiasOutputOffsetWeightScaleMode = params.weightScaleMode; } return fBiasOutputOffset; @@ -624,6 +764,7 @@ struct QuantizedGemmCudaLtState { std::int8_t *InputQuantizedBuffer() const { return fInputQuantized; } std::int32_t *AccumulatorBuffer() const { return fAccumulator; } + void *OutputQuantizedBuffer() const { return fOutputQuantized; } std::size_t AccumulatorBytes() const { return fAccumulatorBytes; } std::size_t WorkspaceSize() const { return fWorkspaceSize; } int HeuristicResultCount() const { return fHeuristicResultCount; } @@ -747,9 +888,11 @@ private: fSelectedCandidateMs = other.fSelectedCandidateMs; fInputQuantized = other.fInputQuantized; fAccumulator = other.fAccumulator; + fOutputQuantized = other.fOutputQuantized; fBiasOutputOffset = other.fBiasOutputOffset; fInputQuantizedBytes = other.fInputQuantizedBytes; fAccumulatorBytes = other.fAccumulatorBytes; + fOutputQuantizedBytes = other.fOutputQuantizedBytes; fBiasOutputOffsetBytes = other.fBiasOutputOffsetBytes; fBiasOutputOffsetSource = other.fBiasOutputOffsetSource; fBiasOutputOffsetBiasScale = other.fBiasOutputOffsetBiasScale; @@ -759,6 +902,8 @@ private: fBiasOutputOffsetBiasQMin = other.fBiasOutputOffsetBiasQMin; fBiasOutputOffsetBiasQMax = other.fBiasOutputOffsetBiasQMax; fBiasOutputOffsetN = other.fBiasOutputOffsetN; + fBiasOutputOffsetWeightScaleVector = other.fBiasOutputOffsetWeightScaleVector; + fBiasOutputOffsetWeightScaleMode = other.fBiasOutputOffsetWeightScaleMode; fM = other.fM; fN = other.fN; fK = other.fK; @@ -785,9 +930,11 @@ private: other.fSelectedCandidateMs = 0.0f; other.fInputQuantized = nullptr; other.fAccumulator = nullptr; + other.fOutputQuantized = nullptr; other.fBiasOutputOffset = nullptr; other.fInputQuantizedBytes = 0; other.fAccumulatorBytes = 0; + other.fOutputQuantizedBytes = 0; other.fBiasOutputOffsetBytes = 0; other.fBiasOutputOffsetSource = nullptr; other.fBiasOutputOffsetBiasScale = 0.0; @@ -797,6 +944,8 @@ private: other.fBiasOutputOffsetBiasQMin = 0; other.fBiasOutputOffsetBiasQMax = 0; other.fBiasOutputOffsetN = 0; + other.fBiasOutputOffsetWeightScaleVector = nullptr; + other.fBiasOutputOffsetWeightScaleMode = EQuantizedCudaScaleMode::PerTensor; other.fM = 0; other.fN = 0; other.fK = 0; @@ -806,6 +955,7 @@ private: inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedGemmCudaStream stream, void *output, const void *input, const void *weight, const float *bias, + const float *weightScaleVector, const QuantizedGemmCudaLtParams ¶ms) { if (output == nullptr || input == nullptr || weight == nullptr) { @@ -820,6 +970,9 @@ inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedG if (params.inputZeroPoint != 0 || params.weightZeroPoint != 0) { throw std::runtime_error("SOFIE cuBLASLt quantized GEMM currently requires input and weight zero points to be 0"); } + if (params.weightScaleMode == EQuantizedCudaScaleMode::PerOutputChannel && weightScaleVector == nullptr) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM per-channel weight scale mode requires a scale vector"); + } if (params.epilogueMode == EQuantizedCudaEpilogueMode::Quantized && params.outputCarrier == EQuantizedCudaOutputCarrier::Float) { throw std::runtime_error("SOFIE cuBLASLt quantized GEMM quantized epilogue requires an integer output carrier"); @@ -830,12 +983,24 @@ inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedG } QuantizedGemmCudaLtParams effectiveParams = params; + if (effectiveParams.logicalM == 0) + effectiveParams.logicalM = effectiveParams.m; + if (effectiveParams.logicalN == 0) + effectiveParams.logicalN = effectiveParams.n; + if (effectiveParams.logicalK == 0) + effectiveParams.logicalK = effectiveParams.k; + if (effectiveParams.paddedExecution && + (effectiveParams.logicalM > effectiveParams.m || effectiveParams.logicalN > effectiveParams.n || + effectiveParams.logicalK > effectiveParams.k)) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM padded execution received logical dimensions larger than physical dimensions"); + } if (effectiveParams.accumulatorToOutputScale == 0.0) effectiveParams.accumulatorToOutputScale = (effectiveParams.inputScale * effectiveParams.weightScale) / effectiveParams.outputScale; const std::size_t inputElements = effectiveParams.m * effectiveParams.k; const std::size_t outputElements = effectiveParams.m * effectiveParams.n; + const std::size_t logicalOutputElements = effectiveParams.logicalM * effectiveParams.logicalN; state.Initialize(effectiveParams); state.EnsureTemporaryBuffers(effectiveParams); QuantizedGemmCudaLtEventTimer profileTimer(QuantizedGemmCudaLt_ProfilingEnabled()); @@ -845,7 +1010,16 @@ inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedG const std::int8_t *inputQuantized = nullptr; if (effectiveParams.inputCarrier == EQuantizedCudaInputCarrier::Int8) { inputQuantized = static_cast(input); + if (effectiveParams.paddedExecution) { + QuantizedGemmCudaPadInt8Matrix(stream, inputQuantized, state.InputQuantizedBuffer(), + effectiveParams.logicalM, effectiveParams.logicalK, + effectiveParams.m, effectiveParams.k, 0); + inputQuantized = state.InputQuantizedBuffer(); + } } else { + if (effectiveParams.paddedExecution) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM padded execution currently requires int8 input carriers"); + } const int inputBlocks = static_cast((inputElements + threads - 1) / threads); const float *inputFloat = static_cast(input); INTERNAL::QuantizedGemmCudaQuantizeInputKernel<<>>( @@ -865,42 +1039,62 @@ inline void QuantizedGemmCudaLt_Call(QuantizedGemmCudaLtState &state, QuantizedG bool directOutputCarrier = false; if (effectiveParams.epilogueMode == EQuantizedCudaEpilogueMode::Quantized) { directOutputCarrier = true; - const float *biasOutputOffset = state.EnsureBiasOutputOffsetBuffer(effectiveParams, bias, stream); + const float *biasOutputOffset = state.EnsureBiasOutputOffsetBuffer(effectiveParams, bias, weightScaleVector, stream); - outputBytes = outputElements * (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8 ? sizeof(std::uint8_t) : sizeof(std::int8_t)); + outputBytes = logicalOutputElements * (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8 ? sizeof(std::uint8_t) : sizeof(std::int8_t)); + void *epilogueOutput = effectiveParams.paddedExecution ? state.OutputQuantizedBuffer() : output; + if (epilogueOutput == nullptr) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM padded execution did not allocate an output scratch buffer"); + } if (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8) { - auto *quantizedOutput = static_cast(output); + auto *quantizedOutput = static_cast(epilogueOutput); if (effectiveParams.hasBias && bias != nullptr) { if (effectiveParams.hasRelu) { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, weightScaleVector, effectiveParams); } else { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, weightScaleVector, effectiveParams); } } else { if (effectiveParams.hasRelu) { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, weightScaleVector, effectiveParams); } else { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, weightScaleVector, effectiveParams); } } } else { - auto *quantizedOutput = static_cast(output); + auto *quantizedOutput = static_cast(epilogueOutput); if (effectiveParams.hasBias && bias != nullptr) { if (effectiveParams.hasRelu) { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, weightScaleVector, effectiveParams); } else { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), biasOutputOffset, weightScaleVector, effectiveParams); } } else { if (effectiveParams.hasRelu) { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, weightScaleVector, effectiveParams); } else { - INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, effectiveParams); + INTERNAL::QuantizedGemmCudaQuantizedEpilogueKernel<<>>(quantizedOutput, state.AccumulatorBuffer(), nullptr, weightScaleVector, effectiveParams); } } } INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaQuantizedEpilogueKernel launch"); + if (effectiveParams.paddedExecution) { + if (effectiveParams.outputCarrier == EQuantizedCudaOutputCarrier::UInt8) { + QuantizedGemmCudaUnpadUInt8Matrix(stream, static_cast(state.OutputQuantizedBuffer()), + static_cast(output), + effectiveParams.logicalM, effectiveParams.logicalN, + effectiveParams.m, effectiveParams.n); + } else { + QuantizedGemmCudaUnpadInt8Matrix(stream, static_cast(state.OutputQuantizedBuffer()), + static_cast(output), + effectiveParams.logicalM, effectiveParams.logicalN, + effectiveParams.m, effectiveParams.n); + } + } } else { + if (effectiveParams.paddedExecution) { + throw std::runtime_error("SOFIE cuBLASLt quantized GEMM padded execution currently requires quantized output mode"); + } auto *floatOutput = static_cast(output); INTERNAL::QuantizedGemmCudaEpilogueKernel<<>>(floatOutput, state.AccumulatorBuffer(), bias, effectiveParams); INTERNAL::CheckCudaStatus(cudaGetLastError(), "QuantizedGemmCudaEpilogueKernel launch"); diff --git a/core/src/RModel.cxx b/core/src/RModel.cxx index 4a070e7..443669f 100644 --- a/core/src/RModel.cxx +++ b/core/src/RModel.cxx @@ -58,6 +58,7 @@ void RModel::BuildLoweredOperatorView(EQuantizedBackend backend) { fLoweredOperators.clear(); fLoweredConsumedOperatorIndices.clear(); + PrepareQuantizedTensorStorage(backend); AddLoweredQuantizedOperators(backend); } @@ -688,9 +689,9 @@ void RModel::Initialize(const std::map & inputParams, bool AnalyzeQuantizedRegions(); - // loop on initialized tensors and make the integers as constant to be - // not written in a weight file and check if the tensors flagged as not writable are really not writable, - // i.e. are not used by non constant operators + // Check whether tensors flagged as not writable are used by runtime operators. + // Integer initializers remain embedded for compatibility, except for physical + // quantized storage explicitly registered as file-backed model weights. for (auto &it : fInitializedTensors) { // check if not-writable tensors are really not writable, i.e. are not used by non constant operators if (it.second.IsNotWritable() && runtimeInitializedInputs.find(it.first) != runtimeInitializedInputs.end()) { @@ -699,9 +700,8 @@ void RModel::Initialize(const std::map & inputParams, bool std::cout << "Initialized tensor " << it.first << " is flagged as not writable but is used by non constant operators, set it as writable \n"; } } - // if the tensor is an integer we can flag it as constant since it will not be written in a weight file and it is considered equivalent as being created from a Constant operator - // only FLOAT tensors are written in a weight file - if (it.second.type() != ETensorType::FLOAT) { + const bool isFileBackedQuantizedStorage = HasQuantizedTensorStorage(it.first); + if (it.second.type() != ETensorType::FLOAT && !isFileBackedQuantizedStorage) { it.second.SetConstant(); } } @@ -811,7 +811,7 @@ void RModel::GenerateInitializedTensorInfo() for (auto &i : fInitializedTensors) { if (i.second.IsNotWritable()) continue; size_t length = ConvertShapeToLength(i.second.shape()); - if (!fUseWeightFile || i.second.IsConstantTensor() || !i.second.IsWeightTensor() || i.second.type() != ETensorType::FLOAT ) { + if (!fUseWeightFile || i.second.IsConstantTensor() || !i.second.IsWeightTensor()) { if (i.second.type() == ETensorType::FLOAT) { // check if NaN of Inf are inside tensor data bool hasInfOrNaN = false; @@ -844,12 +844,14 @@ void RModel::GenerateInitializedTensorInfo() } else { - // case of tensors which are read from a file - if (i.second.type() == ETensorType::FLOAT) { - fGC += "std::vector fTensor_" + i.first + " = std::vector(" + std::to_string(length) + ");\n"; - fGC += "float * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n"; - fWeightsTensorSize += length * sizeof(float); - } + const auto typeName = ConvertTypeToString(i.second.type()); + if (i.second.type() != ETensorType::FLOAT && i.second.type() != ETensorType::INT8 && + i.second.type() != ETensorType::UINT8 && i.second.type() != ETensorType::INT32 && + i.second.type() != ETensorType::INT64) + throw std::runtime_error("sofie initialized tensor " + i.first + " has a type unsupported by weight files"); + fGC += "std::vector<" + typeName + "> fTensor_" + i.first + " = std::vector<" + typeName + ">(" + std::to_string(length) + ");\n"; + fGC += typeName + " * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n"; + fWeightsTensorSize += length * GetTypeSize(i.second.type()); } } } @@ -1599,7 +1601,9 @@ void RModel::ReadInitializedTensorsFromFile(long pos) { // skip Constant and shape tensors (not written in a file) if (!i.second.IsWeightTensor()) continue; std::string tensor_name = "tensor_" + i.first; - if (i.second.type() == ETensorType::FLOAT) { + if (i.second.type() == ETensorType::FLOAT || i.second.type() == ETensorType::INT8 || + i.second.type() == ETensorType::UINT8 || i.second.type() == ETensorType::INT32 || + i.second.type() == ETensorType::INT64) { std::string length = std::to_string(ConvertShapeToLength(i.second.shape())); fGC += " ReadTensorFromStream(f, " + tensor_name + ", \"" + tensor_name + "\", " + length + ");\n"; } else { @@ -1753,6 +1757,30 @@ long RModel::WriteInitializedTensorsToFile(std::string filename) { f << ( (idx < length-1) ? " " : "\n" ); } } + else if (i.second.type() == ETensorType::INT8) { + const int8_t * data = i.second.data(); + for (size_t idx = 0; idx < length; idx++) { + f << static_cast(data[idx]) << ( (idx < length-1) ? " " : "\n" ); + } + } + else if (i.second.type() == ETensorType::UINT8) { + const uint8_t * data = i.second.data(); + for (size_t idx = 0; idx < length; idx++) { + f << static_cast(data[idx]) << ( (idx < length-1) ? " " : "\n" ); + } + } + else if (i.second.type() == ETensorType::INT32) { + const int32_t * data = i.second.data(); + for (size_t idx = 0; idx < length; idx++) { + f << data[idx] << ( (idx < length-1) ? " " : "\n" ); + } + } + else if (i.second.type() == ETensorType::INT64) { + const int64_t * data = i.second.data(); + for (size_t idx = 0; idx < length; idx++) { + f << data[idx] << ( (idx < length-1) ? " " : "\n" ); + } + } else { throw std::runtime_error("sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be written to a file"); } diff --git a/core/src/RModel_ALPAKA.cxx b/core/src/RModel_ALPAKA.cxx index 542190a..b866238 100644 --- a/core/src/RModel_ALPAKA.cxx +++ b/core/src/RModel_ALPAKA.cxx @@ -148,6 +148,8 @@ void RModel::GenerateInitializedTensorInfo_GPU_ALPAKA() { } for (auto &i : fInitializedTensors) { + if (i.second.IsNotWritable()) + continue; if (!fUseWeightFile || i.second.IsConstantTensor()) { if (i.second.type() == ETensorType::FLOAT) fGC += GenerateConstantTensorCode(i); @@ -196,6 +198,8 @@ void RModel::GenerateTemporaryInitializedTensorContainers_GPU_ALPAKA() fGC += "// temporary initialized tensors for loading weights\n"; for (auto &i : fInitializedTensors) { + if (i.second.IsNotWritable()) + continue; if (fUseWeightFile && !i.second.IsConstantTensor()) { // case of tensors which are read from a file size_t length = ConvertShapeToLength(i.second.shape()); @@ -883,7 +887,8 @@ void RModel::MoveInitializedTensorsToBuffers_ALPAKA(){ if (i.second.IsNotWritable()) continue; if (HasQuantizedTensorStorage(i.first)) { const auto &storage = GetQuantizedTensorStorage(i.first); - if (storage.layout == EQuantizedLayout::PackedCPU || !storage.isDeviceResident) + if (storage.layout == EQuantizedLayout::PackedCPU || + storage.residentBackend != EQuantizedBackend::ALPAKA) continue; } const auto type = i.second.type(); diff --git a/core/src/RModel_Quantization.cxx b/core/src/RModel_Quantization.cxx index dc6c22a..98556be 100644 --- a/core/src/RModel_Quantization.cxx +++ b/core/src/RModel_Quantization.cxx @@ -1,17 +1,19 @@ #include "SOFIE/RModel.hxx" #include "SOFIE/ROperator_Gemm.hxx" #include "SOFIE/ROperator_QuantizedGemm.hxx" +#include "SOFIE/ROperator_QuantizedMatMul.hxx" #include "SOFIE/RQuantization.hxx" +#include "SOFIE/RQuantization_Analysis.hxx" +#include "SOFIE/RQuantization_DenseLinear.hxx" +#include "SOFIE/RQuantization_Storage.hxx" #include -#include #include #include #include #include #include #include -#include #include #include @@ -19,63 +21,9 @@ namespace SOFIE { namespace { -std::string JoinReasons(const std::vector &reasons) -{ - std::ostringstream out; - for (std::size_t i = 0; i < reasons.size(); ++i) { - if (i != 0) - out << "; "; - out << reasons[i]; - } - return out.str(); -} - -bool IsScalarPerTensor(const QuantizationInfo &info) -{ - return info.granularity == EQuantizationGranularity::PerTensor && info.axis == -1; -} - -ETensorType TensorTypeForQuantizedStorage(EQuantizedStorageType storage) -{ - switch (storage) { - case EQuantizedStorageType::FloatCarrier: - return ETensorType::FLOAT; - case EQuantizedStorageType::Int8: - return ETensorType::INT8; - case EQuantizedStorageType::UInt8: - return ETensorType::UINT8; - case EQuantizedStorageType::Int32Accumulator: - return ETensorType::INT32; - default: - throw std::runtime_error("SOFIE quantized lowering plan has no physical tensor type for this storage"); - } -} - -std::vector QuantizeTensorToInt8(const float *data, std::size_t length, const QuantizationInfo &info) -{ - std::vector quantized(length); - for (std::size_t i = 0; i < length; ++i) { - quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); - } - return quantized; -} - -std::vector QuantizeTensorToUInt8(const float *data, std::size_t length, const QuantizationInfo &info) -{ - std::vector quantized(length); - for (std::size_t i = 0; i < length; ++i) { - quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); - } - return quantized; -} - std::string GetQuantBoundarySourceTensor(const ROperator &op) { - auto inputs = op.GetOpInputTensors(); - if (inputs.empty()) { - return {}; - } - return std::string(inputs[0]); + return op.GetQuantizationSourceTensor(); } QuantizedGemmCodegenContext MakeQuantizedGemmCodegenContext(const ROperator_Gemm &gemm) @@ -92,301 +40,15 @@ QuantizedGemmCodegenContext MakeQuantizedGemmCodegenContext(const ROperator_Gemm return context; } -std::vector QuantizedGemmConsumedOperatorIndices(const QuantizedGemmRegion ®ion) -{ - std::vector indices = { region.inputQuantOpIndex, region.weightQuantOpIndex, - region.gemmOpIndex, region.outputQuantOpIndex }; - if (region.biasQuantOpIndex) { - indices.push_back(*region.biasQuantOpIndex); - } - std::sort(indices.begin(), indices.end()); - return indices; -} - -QuantizedLoweringPlan MakeAvailableQuantizedGemmPlan(const QuantizedGemmRegion ®ion, - EQuantizedBackend backend, - EQuantizedLoweringStatus status, - std::string reason, - std::string capabilityTag) -{ - QuantizedLoweringPlan plan; - plan.backend = backend; - plan.status = status; - plan.reason = std::move(reason); - plan.capabilityTag = std::move(capabilityTag); - plan.consumedOperatorIndices = QuantizedGemmConsumedOperatorIndices(region); - plan.preservesQuantizationSemantics = true; - plan.isMetadataOnly = false; - plan.suppressesGraphOperators = true; - return plan; -} - -std::vector QuantizedGemmOperatorIndices(const QuantizationModelState &state) -{ - std::vector indices; - indices.reserve(state.gemmRegions.size()); - for (const auto &entry : state.gemmRegions) { - indices.push_back(entry.first); - } - std::sort(indices.begin(), indices.end()); - return indices; -} - -const QuantizedLoweringPlan *FindQuantizedLoweringPlan(const QuantizationModelState &state, - std::size_t opIndex, EQuantizedBackend backend) -{ - auto opIt = state.loweringPlans.find(opIndex); - if (opIt == state.loweringPlans.end()) - return nullptr; - auto backendIt = opIt->second.find(backend); - return backendIt == opIt->second.end() ? nullptr : &backendIt->second; -} - -QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion ®ion, - const std::string &weightStorageTensor) -{ - auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::CPU, EQuantizedLoweringStatus::Baseline, - "CPU baseline lowering with packed pre-quantized weight storage", - "cpu_packed_weight_baseline"); - plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); - plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); - plan.biasStorage = EQuantizedStorageType::FloatCarrier; - plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; - plan.outputStorage = EQuantizedStorageType::FloatCarrier; - plan.inputCarrierMode = EQuantizedCarrierMode::Float; - plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; - plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; - plan.weightStorageTensor = weightStorageTensor; - plan.weightLayout = EQuantizedLayout::PackedCPU; - return plan; -} - -QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend, std::string reason, bool preservesSemantics) -{ - QuantizedLoweringPlan plan; - plan.backend = backend; - plan.status = preservesSemantics ? EQuantizedLoweringStatus::BackendUnsupported - : EQuantizedLoweringStatus::SemanticUnsupported; - plan.reason = std::move(reason); - plan.inputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; - plan.weightStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; - plan.biasStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; - plan.accumulatorStorage = EQuantizedStorageType::UNDEFINED; - plan.outputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; - plan.inputCarrierMode = preservesSemantics ? EQuantizedCarrierMode::Float : EQuantizedCarrierMode::UNDEFINED; - plan.outputMode = preservesSemantics ? EQuantizedOutputMode::ExactFakeQuantFloat : EQuantizedOutputMode::UNDEFINED; - plan.computeProfile = preservesSemantics ? EQuantizedComputeProfile::GenericRecognized : EQuantizedComputeProfile::UNDEFINED; - plan.capabilityTag = preservesSemantics ? "recognized_backend_unsupported" : "semantic_unsupported"; - plan.preservesQuantizationSemantics = preservesSemantics; - plan.isMetadataOnly = preservesSemantics; - plan.suppressesGraphOperators = false; - return plan; -} - -QuantizedLoweringPlan MakeAlpakaFakeQuantPlan(const QuantizedGemmRegion ®ion) -{ - auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Baseline, - "Alpaka fake-quant lowering over float carrier tensors", - "alpaka_fake_quant_baseline"); - plan.inputStorage = EQuantizedStorageType::FloatCarrier; - plan.weightStorage = EQuantizedStorageType::FloatCarrier; - plan.biasStorage = EQuantizedStorageType::FloatCarrier; - plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; - plan.outputStorage = EQuantizedStorageType::FloatCarrier; - plan.inputCarrierMode = EQuantizedCarrierMode::Float; - plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; - plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; - plan.weightLayout = EQuantizedLayout::Plain; - return plan; -} - -struct QuantizedGemmCublasLtCapability { - bool optimized = false; - EQuantizedComputeProfile profile = EQuantizedComputeProfile::GenericRecognized; - std::string tag = "recognized_not_cublaslt_optimized"; - std::string reason; - QuantizedMatMulShapePolicy shapePolicy; -}; - -void AddCapabilityReason(std::vector &reasons, std::string reason) -{ - reasons.push_back(std::move(reason)); -} - -constexpr std::size_t kCublasLtInt8Alignment = 16; -constexpr std::size_t kCublasLtMinOptimizedMacs = 1'000'000; -constexpr double kCublasLtPaddingCandidateMaxWorkRatio = 1.50; - -std::size_t RoundUpToMultiple(std::size_t value, std::size_t multiple) -{ - if (multiple == 0 || value == 0) - return value; - return ((value + multiple - 1) / multiple) * multiple; -} - -bool IsAlignedTo(std::size_t value, std::size_t multiple) -{ - return multiple != 0 && (value % multiple) == 0; -} - -QuantizedMatMulShapePolicy MakeCublasLtShapePolicy(std::size_t m, std::size_t k, std::size_t n) -{ - QuantizedMatMulShapePolicy policy; - policy.logicalM = m; - policy.logicalK = k; - policy.logicalN = n; - policy.physicalM = RoundUpToMultiple(m, kCublasLtInt8Alignment); - policy.physicalK = RoundUpToMultiple(k, kCublasLtInt8Alignment); - policy.physicalN = RoundUpToMultiple(n, kCublasLtInt8Alignment); - - policy.logicalMacs = m * k * n; - policy.physicalMacs = policy.physicalM * policy.physicalK * policy.physicalN; - policy.minimumOptimizedMacs = kCublasLtMinOptimizedMacs; - policy.belowMinimumWork = policy.logicalMacs < policy.minimumOptimizedMacs; - policy.paddingWorkRatio = policy.logicalMacs > 0 ? static_cast(policy.physicalMacs) / - static_cast(policy.logicalMacs) : 1.0; - - std::ostringstream reason; - reason << "logical M/K/N=" << policy.logicalM << "/" << policy.logicalK << "/" << policy.logicalN - << ", physical M/K/N=" << policy.physicalM << "/" << policy.physicalK << "/" << policy.physicalN - << ", logical MACs=" << policy.logicalMacs - << ", physical MACs=" << policy.physicalMacs - << ", minimum optimized MACs=" << policy.minimumOptimizedMacs - << ", padding work ratio=" << policy.paddingWorkRatio; - - if (IsAlignedTo(m, kCublasLtInt8Alignment) && IsAlignedTo(k, kCublasLtInt8Alignment) && - IsAlignedTo(n, kCublasLtInt8Alignment)) { - if (policy.belowMinimumWork) { - policy.policy = EQuantizedShapePolicy::ExactTooSmall; - policy.reason = "exact cuBLASLt int8 shape below minimum optimized work threshold; " + reason.str(); - } else { - policy.policy = EQuantizedShapePolicy::Exact; - policy.reason = "exact cuBLASLt int8 shape; " + reason.str(); - } - } else if (policy.paddingWorkRatio <= kCublasLtPaddingCandidateMaxWorkRatio) { - policy.policy = EQuantizedShapePolicy::PaddedCandidate; - policy.reason = "padded cuBLASLt candidate; " + reason.str(); - } else { - policy.policy = EQuantizedShapePolicy::Fallback; - policy.reason = "padding too expensive for cuBLASLt candidate; " + reason.str(); - } - return policy; -} - -QuantizedGemmCublasLtCapability AssessCublasLtQuantizedGemmCapability( - const QuantizedGemmRegion ®ion, - const std::vector &inputShape, - const std::vector &weightShape) +QuantizedMatrixCodegenContext MakeQuantizedMatMulCodegenContext(const ROperator_Gemm &gemm) { - QuantizedGemmCublasLtCapability capability; - std::vector semanticReasons; - - if (inputShape.size() != 2) - AddCapabilityReason(semanticReasons, "input rank is not 2"); - if (weightShape.size() != 2) - AddCapabilityReason(semanticReasons, "weight rank is not 2"); - - if (inputShape.size() == 2 && weightShape.size() == 2) { - const auto m = inputShape[0]; - const auto k = inputShape[1]; - const auto n = weightShape[0]; - const auto weightK = weightShape[1]; - if (m == 0 || n == 0 || k == 0) - AddCapabilityReason(semanticReasons, "M, N, and K must be nonzero"); - if (k != weightK) - AddCapabilityReason(semanticReasons, "input K does not match weight K"); - if (m != 0 && n != 0 && k != 0 && k == weightK) - capability.shapePolicy = MakeCublasLtShapePolicy(m, k, n); - } - - if (region.inputQuant.bitWidth != 8) - AddCapabilityReason(semanticReasons, "input bit width is not 8"); - if (region.weightQuant.bitWidth != 8) - AddCapabilityReason(semanticReasons, "weight bit width is not 8"); - if (region.outputQuant.bitWidth != 8) - AddCapabilityReason(semanticReasons, "output bit width is not 8"); - if (!region.inputQuant.isSigned) - AddCapabilityReason(semanticReasons, "input quantization is not signed int8"); - if (!region.weightQuant.isSigned) - AddCapabilityReason(semanticReasons, "weight quantization is not signed int8"); - if (region.inputQuant.zeroPoint != 0) - AddCapabilityReason(semanticReasons, "input zero point is not 0"); - if (region.weightQuant.zeroPoint != 0) - AddCapabilityReason(semanticReasons, "weight zero point is not 0"); - if (!IsScalarPerTensor(region.inputQuant)) - AddCapabilityReason(semanticReasons, "input quantization is not per-tensor scalar"); - if (!IsScalarPerTensor(region.weightQuant)) - AddCapabilityReason(semanticReasons, "weight quantization is not per-tensor scalar"); - if (!IsScalarPerTensor(region.outputQuant)) - AddCapabilityReason(semanticReasons, "output quantization is not per-tensor scalar"); - - if (!semanticReasons.empty()) { - capability.shapePolicy.policy = EQuantizedShapePolicy::Unsupported; - capability.shapePolicy.reason = "cuBLASLt semantic requirements are not met"; - capability.reason = JoinReasons(semanticReasons); - capability.tag = "cublaslt_i8i8_semantic_unsupported"; - return capability; - } - - capability.profile = EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; - if (capability.shapePolicy.policy == EQuantizedShapePolicy::Exact) { - capability.optimized = true; - capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_exact"; - capability.reason = "cuBLASLt optimized signed-int8 symmetric per-tensor rank-2 exact-shape GEMM; " + - capability.shapePolicy.reason; - } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::ExactTooSmall) { - capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_exact_too_small"; - capability.reason = "cuBLASLt exact-shape execution is legal but below the minimum optimized work threshold; " + - capability.shapePolicy.reason; - } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::PaddedCandidate) { - capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_padded_candidate"; - capability.reason = "cuBLASLt padded execution is a candidate but is not implemented in this lowering; " + - capability.shapePolicy.reason; - } else { - capability.tag = "cublaslt_i8i8_symmetric_per_tensor_rank2_shape_fallback"; - capability.reason = capability.shapePolicy.reason.empty() ? - "cuBLASLt shape policy is unavailable" : capability.shapePolicy.reason; - } - return capability; -} - - -QuantizedLoweringPlan MakeAlpakaCublasLtCorePlan(const QuantizedGemmRegion ®ion, - const std::string &weightStorageTensor, - const QuantizedGemmCublasLtCapability &capability) -{ - auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Optimized, - capability.reason, capability.tag); - plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); - plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); - plan.biasStorage = EQuantizedStorageType::FloatCarrier; - plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; - plan.outputStorage = QuantizedStorageTypeForCarrier(region.outputQuant); - plan.inputCarrierMode = QuantizedCarrierModeForStorage(plan.inputStorage); - plan.outputMode = EQuantizedOutputMode::Quantized; - plan.computeProfile = EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; - plan.shapePolicy = capability.shapePolicy; - plan.weightStorageTensor = weightStorageTensor; - plan.weightLayout = EQuantizedLayout::PlainDevice; - return plan; + QuantizedMatrixCodegenContext context; + context.inputShape = gemm.GetInputShape(); + context.weightShape = gemm.GetWeightShape(); + context.outputShape = gemm.GetOutputShape(); + return context; } -void CheckQuantInfo(const QuantizationInfo &info, const std::string &role, bool, - std::vector &reasons) -{ - if (info.bitWidth == 0 || info.bitWidth > 8) { - reasons.push_back(role + " bit width is not in the supported QONNX fake-quant range [1, 8]"); - } - if (info.scale <= 0.0 || !std::isfinite(info.scale)) { - reasons.push_back(role + " scale is not positive and finite"); - } - if (info.rounding != EQuantizationRoundingMode::ROUND) { - reasons.push_back(role + " rounding mode is not ROUND"); - } - if (info.overflow != EQuantizationOverflowMode::SAT && info.overflow != EQuantizationOverflowMode::SAT_SYM) { - reasons.push_back(role + " overflow mode is unsupported"); - } -} } // namespace @@ -445,19 +107,55 @@ const QuantizedTensorStorage & RModel::GetQuantizedTensorStorage(const std::stri void RModel::AnalyzeQuantizedRegions() { + for (const auto &[name, storage] : fQuantizationState.tensorStorages) + fInitializedTensors.erase(name); fQuantizationState.ClearDerivedAnalysis(); - std::unordered_map producerByTensor; - std::unordered_map> consumersByTensor; + const auto graph = BuildQuantizationGraphIndex(fOperators); + auto readZeroPointTensor = [this](const std::string &tensorName) { + std::vector values; + auto appendValues = [&values](const auto &typedValues) { + values.reserve(typedValues.size()); + for (auto value : typedValues) + values.push_back(static_cast(value)); + }; - for (std::size_t opIndex = 0; opIndex < fOperators.size(); ++opIndex) { - for (const auto &output : fOperators[opIndex]->GetOpOutputTensors()) { - producerByTensor[std::string(output)] = opIndex; - } - for (const auto &input : fOperators[opIndex]->GetOpInputTensors()) { - consumersByTensor[std::string(input)].push_back(opIndex); + switch (GetTensorType(tensorName)) { + case ETensorType::FLOAT: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::DOUBLE: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::INT8: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::UINT8: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::INT16: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::UINT16: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::INT32: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::UINT32: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::INT64: + appendValues(GetTensorData(tensorName)); + break; + case ETensorType::UINT64: + appendValues(GetTensorData(tensorName)); + break; + default: + throw std::runtime_error("SOFIE quantized lowering expects numeric zero-point tensor [" + tensorName + "]"); } - } + return values; + }; for (std::size_t opIndex = 0; opIndex < fOperators.size(); ++opIndex) { if (fOperators[opIndex]->GetKind() != OperatorKind::GEMM) @@ -467,119 +165,139 @@ void RModel::AnalyzeQuantizedRegions() if (!gemm) continue; - QuantizedGemmRegion info; - info.status = EQuantizedLoweringStatus::SemanticUnsupported; - info.alpha = gemm->GetAlpha(); - info.beta = gemm->GetBeta(); - info.transA = gemm->GetTransA(); - info.transB = gemm->GetTransB(); - info.gemmOpIndex = opIndex; - - std::vector reasons; - - auto inputs = gemm->GetOpInputTensors(); - auto outputs = gemm->GetOpOutputTensors(); - if (inputs.size() < 2 || outputs.size() != 1) { - reasons.push_back("Gemm does not have the expected input/output arity"); - } else { - info.inputTensor = std::string(inputs[0]); - info.weightTensor = std::string(inputs[1]); - if (inputs.size() >= 3) - info.biasTensor = std::string(inputs[2]); - info.gemmOutputTensor = std::string(outputs[0]); - } - - if (std::fabs(info.alpha - 1.0f) > 0.0f) - reasons.push_back("Gemm alpha is not 1"); - if (std::fabs(info.beta - 1.0f) > 0.0f) - reasons.push_back("Gemm beta is not 1"); - if (info.transA != 0) - reasons.push_back("Gemm transA is not 0"); - if (info.transB != 1) - reasons.push_back("Gemm transB is not 1"); - if (!info.inputTensor.empty() && GetTensorShape(info.inputTensor).size() != 2) - reasons.push_back("input tensor is not rank-2 for quantized Gemm lowering"); - if (!info.weightTensor.empty() && GetTensorShape(info.weightTensor).size() != 2) - reasons.push_back("weight tensor is not rank-2 for quantized Gemm lowering"); - if (!info.gemmOutputTensor.empty() && GetTensorShape(info.gemmOutputTensor).size() != 2) - reasons.push_back("Gemm output tensor is not rank-2 for quantized Gemm lowering"); - - const auto requireQuantProducer = [&](const std::string &tensor, const std::string &role) -> std::optional { - auto producer = producerByTensor.find(tensor); - if (producer == producerByTensor.end()) { - reasons.push_back(role + " tensor has no producer quantization boundary"); - return std::nullopt; - } - if (!fOperators[producer->second]->IsQuantizationBoundary()) { - reasons.push_back(role + " tensor producer is not a quantization boundary"); - return std::nullopt; - } - return producer->second; - }; + auto pattern = MatchQuantizedDenseLinearPattern( + *gemm, opIndex, [this](const std::string &tensor) { return GetTensorShape(tensor); }); + auto info = std::move(pattern.region); + auto reasons = std::move(pattern.reasons); + const bool isQuantizedMatMulSpelling = pattern.isMatMul; + const bool isMatMulAddSpelling = pattern.hasInlineMatMulBias; + const bool isMatMulSpelling = pattern.isMatMul && !pattern.hasInlineMatMulBias; + auto matmulShape = std::move(pattern.matmulShape); if (!info.inputTensor.empty()) { - if (auto producer = requireQuantProducer(info.inputTensor, "input")) { + if (auto producer = MatchQuantizationBoundaryProducer(graph, fOperators, info.inputTensor, "input", reasons)) { info.inputQuantOpIndex = *producer; info.inputSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); } if (HasQuantizationInfo(info.inputTensor)) { info.inputQuant = GetQuantizationInfo(info.inputTensor); - CheckQuantInfo(info.inputQuant, "input", false, reasons); + CheckQuantizationInfo(info.inputQuant, "input", reasons); } else { reasons.push_back("input tensor has no QuantizationInfo"); } } if (!info.weightTensor.empty()) { - if (auto producer = requireQuantProducer(info.weightTensor, "weight")) { + if (auto producer = MatchQuantizationBoundaryProducer(graph, fOperators, info.weightTensor, "weight", reasons)) { info.weightQuantOpIndex = *producer; info.weightSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); + if (IsInitializedTensor(info.weightTensor) && GetTensorType(info.weightTensor) == ETensorType::FLOAT) + info.weightSourceTensor = info.weightTensor; } if (HasQuantizationInfo(info.weightTensor)) { info.weightQuant = GetQuantizationInfo(info.weightTensor); - CheckQuantInfo(info.weightQuant, "weight", true, reasons); + CheckQuantizationInfo(info.weightQuant, "weight", reasons); } else { reasons.push_back("weight tensor has no QuantizationInfo"); } } if (!info.biasTensor.empty()) { - if (auto producer = requireQuantProducer(info.biasTensor, "bias")) { - info.biasQuantOpIndex = *producer; - info.biasSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); - } - if (HasQuantizationInfo(info.biasTensor)) { - info.biasQuant = GetQuantizationInfo(info.biasTensor); - CheckQuantInfo(*info.biasQuant, "bias", true, reasons); + if (isMatMulAddSpelling) { + info.biasSourceTensor = info.biasTensor; + if (!IsInitializedTensor(info.biasSourceTensor)) { + reasons.push_back("MatMul fused Add bias must be an initialized constant tensor"); + } else if (!info.gemmOutputTensor.empty() && + !IsDenseLinearBiasLikeShape(GetTensorShape(info.biasSourceTensor), GetTensorShape(info.gemmOutputTensor))) { + reasons.push_back("MatMul fused Add bias is not a dense-linear projection bias broadcast shape"); + } else { + info.biasQuant = MakeAccumulatorBiasQuantization(info.inputQuant, info.weightQuant); + } } else { - reasons.push_back("bias tensor has no QuantizationInfo"); + if (auto producer = MatchQuantizationBoundaryProducer(graph, fOperators, info.biasTensor, "bias", reasons)) { + info.biasQuantOpIndex = *producer; + info.biasSourceTensor = GetQuantBoundarySourceTensor(*fOperators[*producer]); + if (IsInitializedTensor(info.biasTensor) && GetTensorType(info.biasTensor) == ETensorType::FLOAT) + info.biasSourceTensor = info.biasTensor; + } + if (HasQuantizationInfo(info.biasTensor)) { + info.biasQuant = GetQuantizationInfo(info.biasTensor); + CheckQuantizationInfo(*info.biasQuant, "bias", reasons); + } else { + reasons.push_back("bias tensor has no QuantizationInfo"); + } } } + QuantizedEpilogue matmulEpilogue; + if (isMatMulAddSpelling && !info.biasSourceTensor.empty() && info.biasQuant.has_value()) { + matmulEpilogue.kind = EQuantizedEpilogueKind::Bias; + matmulEpilogue.biasSourceTensor = info.biasSourceTensor; + matmulEpilogue.biasQuant = info.biasQuant; + } + if (!info.gemmOutputTensor.empty()) { - auto consumers = consumersByTensor.find(info.gemmOutputTensor); - if (consumers == consumersByTensor.end() || consumers->second.empty()) { + auto consumers = graph.consumersByTensor.find(info.gemmOutputTensor); + if (consumers == graph.consumersByTensor.end() || consumers->second.empty()) { reasons.push_back("Gemm output has no output quantization consumer"); } else if (consumers->second.size() != 1) { reasons.push_back("Gemm output has multiple consumers"); } else { auto consumerIndex = consumers->second.front(); - if (!fOperators[consumerIndex]->IsQuantizationBoundary()) { - reasons.push_back("Gemm output consumer is not a quantization boundary"); - } else { - info.outputQuantOpIndex = consumerIndex; - auto quantOutputs = fOperators[consumerIndex]->GetOpOutputTensors(); + auto setOutputQuantFromBoundary = [&](std::size_t quantIndex) { + info.outputQuantOpIndex = quantIndex; + auto quantOutputs = fOperators[quantIndex]->GetOpOutputTensors(); if (quantOutputs.size() != 1) { reasons.push_back("output quantization boundary does not have exactly one output"); } else { info.outputTensor = std::string(quantOutputs[0]); if (HasQuantizationInfo(info.outputTensor)) { info.outputQuant = GetQuantizationInfo(info.outputTensor); - CheckQuantInfo(info.outputQuant, "output", false, reasons); + CheckQuantizationInfo(info.outputQuant, "output", reasons); } else { reasons.push_back("output tensor has no QuantizationInfo"); } } + }; + + if (fOperators[consumerIndex]->IsQuantizationBoundary()) { + setOutputQuantFromBoundary(consumerIndex); + } else if (isMatMulSpelling && IsFloatAddOperator(*fOperators[consumerIndex])) { + const auto addInputs = fOperators[consumerIndex]->GetOpInputTensors(); + const auto addOutputs = fOperators[consumerIndex]->GetOpOutputTensors(); + if (addInputs.size() != 2 || addOutputs.size() != 1) { + reasons.push_back("MatMul Add epilogue does not have two inputs and one output"); + } else { + const std::string addInputA = std::string(addInputs[0]); + const std::string addInputB = std::string(addInputs[1]); + const std::string biasCandidate = addInputA == info.gemmOutputTensor ? addInputB : + (addInputB == info.gemmOutputTensor ? addInputA : std::string{}); + if (biasCandidate.empty()) { + reasons.push_back("MatMul Add epilogue does not consume the MatMul output"); + } else if (!IsInitializedTensor(biasCandidate)) { + reasons.push_back("MatMul Add epilogue bias must be an initialized constant tensor"); + } else if (!IsDenseLinearBiasLikeShape(GetTensorShape(biasCandidate), GetTensorShape(info.gemmOutputTensor))) { + reasons.push_back("MatMul Add epilogue constant is not a dense-linear projection bias broadcast shape"); + } else { + const std::string addOutput = std::string(addOutputs[0]); + auto addOutputConsumers = graph.consumersByTensor.find(addOutput); + if (addOutputConsumers == graph.consumersByTensor.end() || addOutputConsumers->second.empty()) { + reasons.push_back("MatMul Add epilogue output has no output quantization consumer"); + } else if (addOutputConsumers->second.size() != 1) { + reasons.push_back("MatMul Add epilogue output has multiple consumers"); + } else if (!fOperators[addOutputConsumers->second.front()]->IsQuantizationBoundary()) { + reasons.push_back("MatMul Add epilogue output consumer is not a quantization boundary"); + } else { + matmulEpilogue.kind = EQuantizedEpilogueKind::Bias; + matmulEpilogue.biasSourceTensor = biasCandidate; + matmulEpilogue.biasQuant = MakeAccumulatorBiasQuantization(info.inputQuant, info.weightQuant); + matmulEpilogue.addOpIndex = consumerIndex; + setOutputQuantFromBoundary(addOutputConsumers->second.front()); + } + } + } + } else { + reasons.push_back("Gemm output consumer is not a quantization boundary"); } } } @@ -593,73 +311,156 @@ void RModel::AnalyzeQuantizedRegions() (!info.biasTensor.empty() && HasQuantizationInfo(info.biasTensor)) || (!info.outputTensor.empty() && HasQuantizationInfo(info.outputTensor)); + if (isQuantizedMatMulSpelling) { + if (hasQuantizationEvidence) { + auto matmul = MakeQuantizedMatMulRegionFromGemmLikeRegion(info); + matmul.epilogue = matmulEpilogue; + matmul.shape = matmulShape; + auto &plans = fQuantizationState.loweringPlans[opIndex]; + if (reasons.empty()) { + matmul.status = EQuantizedLoweringStatus::SemanticRecognized; + matmul.reason = QuantizedEpilogueHasBias(matmul.epilogue.kind) + ? "recognized quantized MatMul+Add bias region" + : "recognized quantized MatMul region"; + if (!matmul.shape.reason.empty()) + matmul.reason += "; " + matmul.shape.reason; + + auto cpuReason = matmul.reason + "; CPU QuantizedMatMul lowering is not implemented"; + plans[EQuantizedBackend::CPU] = MakeUnsupportedQuantizedMatMulPlan(matmul, EQuantizedBackend::CPU, cpuReason, true); + + std::vector storageReasons; + std::vector perChannelWeightScales; + const bool perChannelWeight = IsPerChannelAxis(matmul.weightQuant, 1); + + std::vector inputShape; + std::vector weightShape; + std::vector outputShape; + if (!matmul.inputSourceTensor.empty()) + inputShape = GetTensorShape(matmul.inputSourceTensor); + if (!matmul.weightSourceTensor.empty() && IsInitializedTensor(matmul.weightSourceTensor)) + weightShape = GetTensorShape(matmul.weightSourceTensor); + else + storageReasons.push_back("MatMul weight source tensor must be initialized for transposed quantized storage"); + if (!matmul.outputTensor.empty()) + outputShape = GetTensorShape(matmul.outputTensor); + + const auto capability = AssessCublasLtDenseLinearCapability( + MakeDenseLinearOperands(matmul, inputShape, weightShape, outputShape)); + const auto selectedCapability = SelectExecutableDenseLinearCapability(capability); + if (!selectedCapability.optimized) { + storageReasons.push_back("MatMul cuBLASLt optimized profile unavailable: " + selectedCapability.reason); + } + + if (perChannelWeight) { + if (!IsInitializedTensor(matmul.weightQuant.scaleTensor)) { + storageReasons.push_back("MatMul per-channel weight scale tensor is not initialized"); + } else if (weightShape.size() == 2) { + perChannelWeightScales = GetTensorData(matmul.weightQuant.scaleTensor); + if (perChannelWeightScales.size() != weightShape[1]) { + storageReasons.push_back("MatMul per-channel weight scale length does not match output channels N"); + } + } + if (!IsInitializedTensor(matmul.weightQuant.zeroPointTensor)) { + storageReasons.push_back("MatMul per-channel weight zero-point tensor is not initialized"); + } else { + const auto zeroPoints = readZeroPointTensor(matmul.weightQuant.zeroPointTensor); + if (weightShape.size() == 2 && zeroPoints.size() != weightShape[1]) { + storageReasons.push_back("MatMul per-channel weight zero-point length does not match output channels N"); + } + for (std::int64_t zeroPoint : zeroPoints) { + if (zeroPoint != 0) { + storageReasons.push_back("MatMul per-channel weight zero-points must all be 0"); + break; + } + } + } + } + + if (storageReasons.empty()) { + const bool paddedStorage = selectedCapability.shapePolicy.policy == EQuantizedShapePolicy::Padded; + const auto deviceStorageTensor = matmul.weightSourceTensor + + (paddedStorage ? "_s22_matmul_transposed_padded_plain_device_storage" + : "_s19_matmul_transposed_plain_device_storage"); + auto alpakaPlan = MakeMatMulAlpakaTransposedWeightStoragePlan(matmul, deviceStorageTensor, selectedCapability.shapePolicy); + alpakaPlan.computeProfile = selectedCapability.profile; + alpakaPlan.capabilityTag = selectedCapability.tag; + alpakaPlan.reason = matmul.reason + "; " + selectedCapability.reason; + plans[EQuantizedBackend::ALPAKA] = std::move(alpakaPlan); + matmul.reason += "; transposed pre-quantized ALPAKA weight storage selected"; + } else { + auto alpakaReason = matmul.reason + "; " + JoinQuantizationReasons(storageReasons); + auto unsupportedPlan = MakeUnsupportedQuantizedMatMulPlan(matmul, EQuantizedBackend::ALPAKA, alpakaReason, true); + unsupportedPlan.capabilityTag = capability.tag; + unsupportedPlan.computeProfile = capability.profile; + unsupportedPlan.shapePolicy = capability.shapePolicy; + plans[EQuantizedBackend::ALPAKA] = std::move(unsupportedPlan); + matmul.reason = alpakaReason; + } + } else { + matmul.status = EQuantizedLoweringStatus::SemanticUnsupported; + matmul.reason = JoinQuantizationReasons(reasons); + plans[EQuantizedBackend::CPU] = MakeUnsupportedQuantizedMatMulPlan(matmul, EQuantizedBackend::CPU, matmul.reason, false); + plans[EQuantizedBackend::ALPAKA] = MakeUnsupportedQuantizedMatMulPlan(matmul, EQuantizedBackend::ALPAKA, matmul.reason, false); + } + fQuantizationState.matmulRegions[opIndex] = std::move(matmul); + if (fVerbose > 0) { + std::cout << "SOFIE quantized MatMul candidate at operator " << opIndex << ": " + << fQuantizationState.matmulRegions[opIndex].reason << std::endl; + } + } + continue; + } + if (reasons.empty()) { info.status = EQuantizedLoweringStatus::SemanticRecognized; info.reason = "recognized quantized Gemm region"; + + auto currentLoweringUnsupportedReasons = QuantizedGemmLoweringUnsupportedReasons(info); + std::vector perChannelWeightScales; + if (IsPerChannelAxis(info.weightQuant, 0)) { + if (!IsInitializedTensor(info.weightQuant.scaleTensor)) { + currentLoweringUnsupportedReasons.push_back("per-channel weight scale tensor is not initialized"); + } else { + perChannelWeightScales = GetTensorData(info.weightQuant.scaleTensor); + const auto weightShape = GetTensorShape(info.weightSourceTensor); + if (weightShape.size() != 2 || perChannelWeightScales.size() != weightShape[0]) { + currentLoweringUnsupportedReasons.push_back("per-channel weight scale length does not match GEMM output channels"); + } + } + if (!IsInitializedTensor(info.weightQuant.zeroPointTensor)) { + currentLoweringUnsupportedReasons.push_back("per-channel weight zero-point tensor is not initialized"); + } else { + const auto zeroPoints = readZeroPointTensor(info.weightQuant.zeroPointTensor); + for (std::int64_t zeroPoint : zeroPoints) { + if (zeroPoint != 0) { + currentLoweringUnsupportedReasons.push_back("per-channel weight zero-points must all be 0"); + break; + } + } + } + } + if (!currentLoweringUnsupportedReasons.empty()) { + info.reason += "; " + JoinQuantizationReasons(currentLoweringUnsupportedReasons); + auto &plans = fQuantizationState.loweringPlans[opIndex]; + plans[EQuantizedBackend::CPU] = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::CPU, info.reason, true); + plans[EQuantizedBackend::ALPAKA] = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::ALPAKA, info.reason, true); + fQuantizationState.gemmRegions[opIndex] = std::move(info); + if (fVerbose > 0) { + std::cout << "SOFIE quantized Gemm candidate recognized but not lowered at operator " << opIndex << ": " + << fQuantizationState.gemmRegions[opIndex].reason << std::endl; + } + continue; + } + auto cpuPlan = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::CPU, "CPU quantized Gemm lowering requires constant pre-quantized weight storage", true); auto alpakaPlan = MakeAlpakaFakeQuantPlan(info); - if (!info.weightSourceTensor.empty() && IsInitializedTensor(info.weightSourceTensor)) { - constexpr std::size_t packedTileN = 4; + if (!IsPerChannelAxis(info.weightQuant, 0) && !info.weightSourceTensor.empty() && IsInitializedTensor(info.weightSourceTensor)) { const auto storageTensor = info.weightSourceTensor + "_s11_packed_cpu_storage"; const auto weightShape = GetTensorShape(info.weightSourceTensor); if (weightShape.size() != 2) { reasons.push_back("weight tensor is not rank-2 for packed CPU storage"); } else { - const auto n = weightShape[0]; - const auto k = weightShape[1]; - const auto packedBlocks = (n + packedTileN - 1) / packedTileN; - const std::vector packedShape = { packedBlocks, k, packedTileN }; - const auto packedLength = ConvertShapeToLength(packedShape); - const float *weightData = fInitializedTensors.at(info.weightSourceTensor).data(); - - if (info.weightQuant.isSigned) { - std::vector packedWeights(packedLength, 0); - for (std::size_t block = 0; block < packedBlocks; ++block) { - for (std::size_t kk = 0; kk < k; ++kk) { - for (std::size_t ji = 0; ji < packedTileN; ++ji) { - const auto col = block * packedTileN + ji; - if (col < n) { - packedWeights[(block * k + kk) * packedTileN + ji] = - static_cast(QuantizeScalarToIntegerGrid(weightData[col * k + kk], info.weightQuant)); - } - } - } - } - if (!IsInitializedTensor(storageTensor)) { - AddConstantTensor(storageTensor, packedShape, packedWeights); - } - } else { - std::vector packedWeights(packedLength, 0); - for (std::size_t block = 0; block < packedBlocks; ++block) { - for (std::size_t kk = 0; kk < k; ++kk) { - for (std::size_t ji = 0; ji < packedTileN; ++ji) { - const auto col = block * packedTileN + ji; - if (col < n) { - packedWeights[(block * k + kk) * packedTileN + ji] = - static_cast(QuantizeScalarToIntegerGrid(weightData[col * k + kk], info.weightQuant)); - } - } - } - } - if (!IsInitializedTensor(storageTensor)) { - AddConstantTensor(storageTensor, packedShape, packedWeights); - } - } - - QuantizedTensorStorage storage; - storage.logicalTensor = info.weightTensor; - storage.sourceTensor = info.weightSourceTensor; - storage.storageTensor = storageTensor; - storage.storageType = QuantizedStorageTypeForCarrier(info.weightQuant); - storage.layout = EQuantizedLayout::PackedCPU; - storage.quantization = info.weightQuant; - storage.shape = packedShape; - storage.residentBackend = EQuantizedBackend::CPU; - storage.isConstant = true; - storage.isDeviceResident = false; - RegisterQuantizedTensorStorage(std::move(storage)); - cpuPlan = MakeCPUPackedWeightBaselinePlan(info, storageTensor); } } @@ -667,43 +468,29 @@ void RModel::AnalyzeQuantizedRegions() if (!info.weightSourceTensor.empty() && IsInitializedTensor(info.weightSourceTensor)) { const auto deviceStorageTensor = info.weightSourceTensor + "_s17g3_plain_device_storage"; const auto weightShape = GetTensorShape(info.weightSourceTensor); - const auto weightLength = ConvertShapeToLength(weightShape); - const float *weightData = fInitializedTensors.at(info.weightSourceTensor).data(); - - if (!IsInitializedTensor(deviceStorageTensor)) { - if (info.weightQuant.isSigned) { - AddConstantTensor(deviceStorageTensor, weightShape, QuantizeTensorToInt8(weightData, weightLength, info.weightQuant)); - } else { - AddConstantTensor(deviceStorageTensor, weightShape, QuantizeTensorToUInt8(weightData, weightLength, info.weightQuant)); - } - } - - QuantizedTensorStorage deviceStorage; - deviceStorage.logicalTensor = info.weightTensor; - deviceStorage.sourceTensor = info.weightSourceTensor; - deviceStorage.storageTensor = deviceStorageTensor; - deviceStorage.storageType = QuantizedStorageTypeForCarrier(info.weightQuant); - deviceStorage.layout = EQuantizedLayout::PlainDevice; - deviceStorage.quantization = info.weightQuant; - deviceStorage.shape = weightShape; - deviceStorage.residentBackend = EQuantizedBackend::ALPAKA; - deviceStorage.isConstant = true; - deviceStorage.isDeviceResident = true; - RegisterQuantizedTensorStorage(std::move(deviceStorage)); try { const auto inputShape = GetTensorShape(info.inputSourceTensor); - const auto capability = AssessCublasLtQuantizedGemmCapability(info, inputShape, weightShape); - if (capability.optimized) { - alpakaPlan = MakeAlpakaCublasLtCorePlan(info, deviceStorageTensor, capability); + const auto outputShape = GetTensorShape(info.outputTensor); + auto capability = AssessCublasLtDenseLinearCapability( + MakeDenseLinearOperands(info, inputShape, weightShape, outputShape)); + auto selectedCapability = SelectExecutableDenseLinearCapability(capability); + if (selectedCapability.optimized) { + std::string selectedStorageTensor = deviceStorageTensor; + if (selectedCapability.shapePolicy.policy == EQuantizedShapePolicy::Padded) { + const auto paddedStorageTensor = info.weightSourceTensor + "_s22_gemm_padded_plain_device_storage"; + selectedStorageTensor = paddedStorageTensor; + } + alpakaPlan = MakeAlpakaCublasLtCorePlan(info, selectedStorageTensor, selectedCapability); } else { alpakaPlan.reason += "; cuBLASLt optimized profile unavailable: " + capability.reason; alpakaPlan.capabilityTag = capability.tag; alpakaPlan.computeProfile = capability.profile; alpakaPlan.shapePolicy = capability.shapePolicy; } - } catch (const std::exception &) { - // Dynamic or unavailable input shapes keep the Alpaka fake-quant baseline plan. + } catch (const std::exception &e) { + alpakaPlan.reason += "; cuBLASLt optimized profile unavailable: " + std::string(e.what()); + alpakaPlan.capabilityTag = "cublaslt_shape_unavailable"; } } @@ -712,7 +499,7 @@ void RModel::AnalyzeQuantizedRegions() plans[EQuantizedBackend::ALPAKA] = std::move(alpakaPlan); fQuantizationState.gemmRegions[opIndex] = std::move(info); } else { - info.reason = JoinReasons(reasons); + info.reason = JoinQuantizationReasons(reasons); if (hasQuantizationEvidence) { auto &plans = fQuantizationState.loweringPlans[opIndex]; plans[EQuantizedBackend::CPU] = MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend::CPU, info.reason, true); @@ -726,6 +513,135 @@ void RModel::AnalyzeQuantizedRegions() } } +void RModel::PrepareQuantizedTensorStorage(EQuantizedBackend backend) +{ + for (const auto &[name, storage] : fQuantizationState.tensorStorages) + fInitializedTensors.erase(name); + fQuantizationState.tensorStorages.clear(); + + auto restoreSource = [this](const std::string &name) { + auto it = fInitializedTensors.find(name); + if (it != fInitializedTensors.end()) + it->second.SetWritable(); + }; + for (const auto &[index, region] : fQuantizationState.gemmRegions) + restoreSource(region.weightSourceTensor); + for (const auto &[index, region] : fQuantizationState.matmulRegions) + restoreSource(region.weightSourceTensor); + + auto installStorage = [this](MaterializedQuantizedWeight materialized) { + const auto name = materialized.storage.storageTensor; + const auto shape = materialized.storage.shape; + std::visit([this, &name, &shape](auto &&buffer) { + AddInitializedTensor(name, shape, std::forward(buffer)); + }, std::move(materialized.buffer)); + RegisterQuantizedTensorStorage(std::move(materialized.storage)); + }; + + for (auto opIndex : SortedQuantizedRegionOperatorIndices(fQuantizationState.gemmRegions)) { + const auto *plan = FindQuantizedLoweringPlan(fQuantizationState, opIndex, backend); + if (plan == nullptr || !IsQuantizedLoweringAvailable(plan->status) || plan->weightStorageTensor.empty()) + continue; + + const auto regionIt = fQuantizationState.gemmRegions.find(opIndex); + if (regionIt == fQuantizationState.gemmRegions.end()) + throw std::runtime_error("SOFIE quantized Gemm storage plan has no matching region"); + const auto ®ion = regionIt->second; + const auto weightShape = GetTensorShape(region.weightSourceTensor); + if (weightShape.size() != 2 || !IsInitializedTensor(region.weightSourceTensor)) + throw std::runtime_error("SOFIE quantized Gemm storage requires an initialized rank-2 weight tensor"); + const auto *weightData = fInitializedTensors.at(region.weightSourceTensor).data(); + + std::vector perChannelScales; + if (backend == EQuantizedBackend::ALPAKA && IsPerChannelAxis(region.weightQuant, 0)) + perChannelScales = GetTensorData(region.weightQuant.scaleTensor); + installStorage(MaterializeQuantizedGemmWeight(region, *plan, backend, weightData, + weightShape, perChannelScales)); + } + + for (auto opIndex : SortedQuantizedRegionOperatorIndices(fQuantizationState.matmulRegions)) { + const auto *plan = FindQuantizedLoweringPlan(fQuantizationState, opIndex, backend); + if (plan == nullptr || !IsQuantizedLoweringAvailable(plan->status) || plan->weightStorageTensor.empty()) + continue; + if (backend != EQuantizedBackend::ALPAKA) + continue; + + const auto regionIt = fQuantizationState.matmulRegions.find(opIndex); + if (regionIt == fQuantizationState.matmulRegions.end()) + throw std::runtime_error("SOFIE quantized MatMul storage plan has no matching region"); + const auto ®ion = regionIt->second; + const auto weightShape = GetTensorShape(region.weightSourceTensor); + if (weightShape.size() != 2 || !IsInitializedTensor(region.weightSourceTensor)) + throw std::runtime_error("SOFIE quantized MatMul storage requires an initialized rank-2 weight tensor"); + + const auto *weightData = fInitializedTensors.at(region.weightSourceTensor).data(); + std::vector perChannelScales; + if (IsPerChannelAxis(region.weightQuant, 1)) + perChannelScales = GetTensorData(region.weightQuant.scaleTensor); + + installStorage(MaterializeQuantizedMatMulWeight(region, *plan, backend, weightData, + weightShape, perChannelScales)); + } + + std::unordered_set consumedOperators; + std::unordered_set pruneCandidates; + std::unordered_set protectedTensors; + for (const auto &[opIndex, backendPlans] : fQuantizationState.loweringPlans) { + auto planIt = backendPlans.find(backend); + if (planIt == backendPlans.end() || !IsQuantizedLoweringAvailable(planIt->second.status) || + planIt->second.weightStorageTensor.empty()) + continue; + consumedOperators.insert(planIt->second.consumedOperatorIndices.begin(), + planIt->second.consumedOperatorIndices.end()); + protectedTensors.insert(planIt->second.weightStorageTensor); + if (!planIt->second.weightScaleTensor.empty()) + protectedTensors.insert(planIt->second.weightScaleTensor); + if (!planIt->second.weightZeroPointTensor.empty()) + protectedTensors.insert(planIt->second.weightZeroPointTensor); + if (auto gemm = fQuantizationState.gemmRegions.find(opIndex); gemm != fQuantizationState.gemmRegions.end()) { + pruneCandidates.insert(gemm->second.weightSourceTensor); + if (!gemm->second.biasSourceTensor.empty()) + protectedTensors.insert(gemm->second.biasSourceTensor); + } + if (auto matmul = fQuantizationState.matmulRegions.find(opIndex); matmul != fQuantizationState.matmulRegions.end()) { + pruneCandidates.insert(matmul->second.weightSourceTensor); + if (!matmul->second.epilogue.biasSourceTensor.empty()) + protectedTensors.insert(matmul->second.epilogue.biasSourceTensor); + } + } + + for (auto opIndex : consumedOperators) { + if (opIndex >= fOperators.size()) + continue; + for (const auto &input : fOperators[opIndex]->GetOpInputTensors()) { + const auto name = UTILITY::Clean_name(std::string(input)); + if (fInitializedTensors.find(name) != fInitializedTensors.end()) + pruneCandidates.insert(name); + } + } + + for (const auto &source : pruneCandidates) { + auto tensor = fInitializedTensors.find(source); + if (tensor == fInitializedTensors.end() || protectedTensors.count(source) != 0) + continue; + + bool hasLiveConsumer = false; + for (std::size_t opIndex = 0; opIndex < fOperators.size() && !hasLiveConsumer; ++opIndex) { + for (const auto &input : fOperators[opIndex]->GetOpInputTensors()) { + if (UTILITY::Clean_name(std::string(input)) == source && consumedOperators.count(opIndex) == 0) { + hasLiveConsumer = true; + break; + } + } + } + if (std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), source) != fOutputTensorNames.end()) + hasLiveConsumer = true; + + if (!hasLiveConsumer) + tensor->second.SetNotWritable(); + } +} + void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) { auto setKnownTensorType = [this](const std::string &tensorName, ETensorType type) { @@ -746,7 +662,26 @@ void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) } }; - for (auto op_idx : QuantizedGemmOperatorIndices(fQuantizationState)) { + auto installLoweredOperator = [this, &setKnownTensorType](std::size_t opIndex, + const QuantizedLoweringPlan &plan, + const std::string &inputSourceTensor, + const std::string &outputTensor, + std::unique_ptr lowered) { + if (QuantizedPlanExposesQuantizedInputCarrier(plan)) + setKnownTensorType(inputSourceTensor, TensorTypeForQuantizedStorage(plan.inputStorage)); + if (QuantizedPlanExposesQuantizedOutputCarrier(plan)) + setKnownTensorType(outputTensor, TensorTypeForQuantizedStorage(plan.outputStorage)); + + fLoweredOperators[opIndex] = std::move(lowered); + if (!plan.suppressesGraphOperators) + return; + for (auto consumedOpIndex : plan.consumedOperatorIndices) { + if (consumedOpIndex != opIndex) + fLoweredConsumedOperatorIndices.insert(consumedOpIndex); + } + }; + + for (auto op_idx : SortedQuantizedRegionOperatorIndices(fQuantizationState.gemmRegions)) { const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); if (planPtr == nullptr) continue; @@ -763,38 +698,44 @@ void RModel::AddLoweredQuantizedOperators(EQuantizedBackend backend) if (regionIt == fQuantizationState.gemmRegions.end()) throw std::runtime_error("SOFIE quantized Gemm lowering plan has no matching region"); const auto ®ion = regionIt->second; - if (QuantizedPlanExposesQuantizedInputCarrier(plan)) { - setKnownTensorType(region.inputSourceTensor, TensorTypeForQuantizedStorage(plan.inputStorage)); - } - if (QuantizedPlanExposesQuantizedOutputCarrier(plan)) { - setKnownTensorType(region.outputTensor, TensorTypeForQuantizedStorage(plan.outputStorage)); - } + installLoweredOperator(op_idx, plan, region.inputSourceTensor, region.outputTensor, + std::make_unique(region, plan, MakeQuantizedGemmCodegenContext(*gemm))); + } + + for (auto op_idx : SortedQuantizedRegionOperatorIndices(fQuantizationState.matmulRegions)) { + const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); + if (planPtr == nullptr) + continue; - fLoweredOperators[op_idx] = std::make_unique( - region, plan, MakeQuantizedGemmCodegenContext(*gemm)); + auto *gemm = dynamic_cast *>(fOperators[op_idx].get()); + if (!gemm) + throw std::runtime_error("SOFIE quantized MatMul region is attached to a non-float Gemm-spelled MatMul operator"); - if (plan.suppressesGraphOperators) { - for (auto consumedOpIndex : plan.consumedOperatorIndices) { - if (consumedOpIndex != op_idx) - fLoweredConsumedOperatorIndices.insert(consumedOpIndex); - } - } + const auto &plan = *planPtr; + if (!IsQuantizedLoweringAvailable(plan.status)) + continue; + + const auto regionIt = fQuantizationState.matmulRegions.find(op_idx); + if (regionIt == fQuantizationState.matmulRegions.end()) + throw std::runtime_error("SOFIE quantized MatMul lowering plan has no matching region"); + const auto ®ion = regionIt->second; + installLoweredOperator(op_idx, plan, region.inputSourceTensor, region.outputTensor, + std::make_unique(region, plan, MakeQuantizedMatMulCodegenContext(*gemm))); } } void RModel::AddQuantizedGeneratedHeaders(EQuantizedBackend backend) { - for (auto op_idx : QuantizedGemmOperatorIndices(fQuantizationState)) { - const auto *planPtr = FindQuantizedLoweringPlan(fQuantizationState, op_idx, backend); - if (planPtr == nullptr) + for (const auto &[opIndex, backendPlans] : fQuantizationState.loweringPlans) { + auto planIt = backendPlans.find(backend); + if (planIt == backendPlans.end()) continue; - const auto &plan = *planPtr; + const auto &plan = planIt->second; if (!IsQuantizedLoweringAvailable(plan.status) || !plan.suppressesGraphOperators) continue; if (backend == EQuantizedBackend::CPU && QuantizedPlanUsesPrequantizedWeights(plan) && - plan.weightLayout == EQuantizedLayout::PackedCPU) { + plan.weightLayout == EQuantizedLayout::PackedCPU) AddNeededCustomHeader("SOFIE/SOFIE_Quantized.hxx"); - } if (backend == EQuantizedBackend::ALPAKA && IsOptimizedQuantizedAlpakaPlainDevicePlan(plan)) { AddNeededCustomHeader("SOFIE/SOFIE_QuantizedAlpaka.hxx"); } diff --git a/core/src/RQuantization_Analysis.cxx b/core/src/RQuantization_Analysis.cxx new file mode 100644 index 0000000..f9ba65d --- /dev/null +++ b/core/src/RQuantization_Analysis.cxx @@ -0,0 +1,241 @@ +#include "SOFIE/RQuantization_Analysis.hxx" +#include "SOFIE/ROperator_BasicBinary.hxx" +#include "SOFIE/RQuantization_DenseLinear.hxx" + +#include +#include +#include + +namespace SOFIE { + +QuantizedDenseLinearPatternMatch MatchQuantizedDenseLinearPattern( + const ROperator_Gemm &gemm, std::size_t opIndex, + const std::function(const std::string &)> &tensorShape) +{ + QuantizedDenseLinearPatternMatch match; + auto ®ion = match.region; + region.status = EQuantizedLoweringStatus::SemanticUnsupported; + region.alpha = gemm.GetAlpha(); + region.beta = gemm.GetBeta(); + region.transA = gemm.GetTransA(); + region.transB = gemm.GetTransB(); + region.gemmOpIndex = opIndex; + + const auto inputs = gemm.GetOpInputTensors(); + const auto outputs = gemm.GetOpOutputTensors(); + if (inputs.size() < 2 || outputs.size() != 1) { + match.reasons.push_back("Gemm does not have the expected input/output arity"); + } else { + region.inputTensor = std::string(inputs[0]); + region.weightTensor = std::string(inputs[1]); + if (inputs.size() >= 3) + region.biasTensor = std::string(inputs[2]); + region.gemmOutputTensor = std::string(outputs[0]); + } + + const bool matmul = inputs.size() == 2 && region.alpha == 1.0f && region.beta == 0.0f && + region.transA == 0 && region.transB == 0; + match.hasInlineMatMulBias = inputs.size() == 3 && region.alpha == 1.0f && region.beta == 1.0f && + region.transA == 0 && region.transB == 0; + match.isMatMul = matmul || match.hasInlineMatMulBias; + if (match.isMatMul) { + const auto inputShape = region.inputTensor.empty() ? std::vector{} : tensorShape(region.inputTensor); + const auto weightShape = region.weightTensor.empty() ? std::vector{} : tensorShape(region.weightTensor); + const auto outputShape = region.gemmOutputTensor.empty() ? std::vector{} : tensorShape(region.gemmOutputTensor); + match.matmulShape = AssessQuantizedMatMulShape(inputShape, weightShape, outputShape); + if (!QuantizedMatMulShapeIsRecognized(match.matmulShape)) { + match.reasons.push_back(match.matmulShape.reason.empty() + ? "MatMul shape is not recognized by quantized dense-linear analysis" + : match.matmulShape.reason); + } + } else { + CheckQuantizedGemmAttributes(region, match.reasons); + if (!region.inputTensor.empty()) + CheckQuantizedGemmRank2Shape(tensorShape(region.inputTensor), "input", match.reasons); + if (!region.weightTensor.empty()) + CheckQuantizedGemmRank2Shape(tensorShape(region.weightTensor), "weight", match.reasons); + if (!region.gemmOutputTensor.empty()) + CheckQuantizedGemmRank2Shape(tensorShape(region.gemmOutputTensor), "Gemm output", match.reasons); + } + return match; +} + +QuantizationGraphIndex BuildQuantizationGraphIndex(const std::vector> &operators) +{ + QuantizationGraphIndex graph; + for (std::size_t opIndex = 0; opIndex < operators.size(); ++opIndex) { + for (const auto &output : operators[opIndex]->GetOpOutputTensors()) + graph.producerByTensor[std::string(output)] = opIndex; + for (const auto &input : operators[opIndex]->GetOpInputTensors()) + graph.consumersByTensor[std::string(input)].push_back(opIndex); + } + return graph; +} + +std::optional MatchQuantizationBoundaryProducer( + const QuantizationGraphIndex &graph, const std::vector> &operators, + const std::string &tensor, const std::string &role, std::vector &reasons) +{ + auto producer = graph.producerByTensor.find(tensor); + if (producer == graph.producerByTensor.end()) { + reasons.push_back(role + " tensor has no producer quantization boundary"); + return std::nullopt; + } + if (!operators[producer->second]->IsQuantizationBoundary()) { + reasons.push_back(role + " tensor producer is not a quantization boundary"); + return std::nullopt; + } + return producer->second; +} + +std::optional MatchSingleTensorConsumer(const QuantizationGraphIndex &graph, + const std::string &tensor, + const std::string &role, + std::vector &reasons) +{ + auto consumers = graph.consumersByTensor.find(tensor); + if (consumers == graph.consumersByTensor.end() || consumers->second.empty()) { + reasons.push_back(role + " has no consumer"); + return std::nullopt; + } + if (consumers->second.size() != 1) { + reasons.push_back(role + " has multiple consumers"); + return std::nullopt; + } + return consumers->second.front(); +} + +bool IsFloatAddOperator(const ROperator &op) +{ + return dynamic_cast *>(&op) != nullptr; +} + +void CheckQuantizationInfo(const QuantizationInfo &info, const std::string &role, + std::vector &reasons) +{ + if (info.bitWidth == 0 || info.bitWidth > 8) + reasons.push_back(role + " bit width is not in the supported quantized integer range [1, 8]"); + if (info.scale <= 0.0 || !std::isfinite(info.scale)) + reasons.push_back(role + " scale is not positive and finite"); + if (info.rounding != EQuantizationRoundingMode::ROUND) + reasons.push_back(role + " rounding mode is not ROUND"); + if (info.overflow != EQuantizationOverflowMode::SAT && info.overflow != EQuantizationOverflowMode::SAT_SYM) + reasons.push_back(role + " overflow mode is unsupported"); +} + +void CheckQuantizedGemmAttributes(const QuantizedGemmRegion ®ion, + std::vector &reasons) +{ + if (std::fabs(region.alpha - 1.0f) > 0.0f) + reasons.push_back("Gemm alpha is not 1"); + if (std::fabs(region.beta - 1.0f) > 0.0f) + reasons.push_back("Gemm beta is not 1"); + if (region.transA != 0) + reasons.push_back("Gemm transA is not 0"); + if (region.transB != 1) + reasons.push_back("Gemm transB is not 1"); +} + +void CheckQuantizedGemmRank2Shape(const std::vector &shape, + const std::string &role, + std::vector &reasons) +{ + if (shape.size() != 2) + reasons.push_back(role + " tensor is not rank-2 for quantized Gemm lowering"); +} + +std::vector QuantizedGemmLoweringUnsupportedReasons(const QuantizedGemmRegion ®ion) +{ + return DenseLinearQuantizationParameterUnsupportedReasons(region.inputQuant, region.weightQuant, + region.outputQuant, region.biasQuant, 0, + "quantized Gemm lowering"); +} + +std::string JoinQuantizationReasons(const std::vector &reasons) +{ + std::ostringstream out; + for (std::size_t i = 0; i < reasons.size(); ++i) { + if (i != 0) + out << "; "; + out << reasons[i]; + } + return out.str(); +} + +std::vector QuantizedGemmConsumedOperatorIndices(const QuantizedGemmRegion ®ion) +{ + std::vector indices = {region.inputQuantOpIndex, region.weightQuantOpIndex, + region.gemmOpIndex, region.outputQuantOpIndex}; + if (region.biasQuantOpIndex) + indices.push_back(*region.biasQuantOpIndex); + std::sort(indices.begin(), indices.end()); + return indices; +} + +std::vector QuantizedMatMulConsumedOperatorIndices(const QuantizedMatMulRegion ®ion) +{ + std::vector indices = {region.inputQuantOpIndex, region.weightQuantOpIndex, + region.matmulOpIndex, region.outputQuantOpIndex}; + if (region.epilogue.addOpIndex) + indices.push_back(*region.epilogue.addOpIndex); + std::sort(indices.begin(), indices.end()); + return indices; +} + +QuantizedMatMulRegion MakeQuantizedMatMulRegionFromGemmLikeRegion(const QuantizedGemmRegion ®ion) +{ + QuantizedMatMulRegion matmul; + matmul.inputTensor = region.inputTensor; + matmul.weightTensor = region.weightTensor; + matmul.matmulOutputTensor = region.gemmOutputTensor; + matmul.outputTensor = region.outputTensor; + matmul.inputSourceTensor = region.inputSourceTensor; + matmul.weightSourceTensor = region.weightSourceTensor; + matmul.inputQuantOpIndex = region.inputQuantOpIndex; + matmul.weightQuantOpIndex = region.weightQuantOpIndex; + matmul.matmulOpIndex = region.gemmOpIndex; + matmul.outputQuantOpIndex = region.outputQuantOpIndex; + matmul.inputQuant = region.inputQuant; + matmul.weightQuant = region.weightQuant; + matmul.outputQuant = region.outputQuant; + return matmul; +} + +bool IsDenseLinearBiasLikeShape(const std::vector &biasShape, + const std::vector &outputShape) +{ + if (outputShape.empty() || biasShape.empty()) + return false; + const auto n = outputShape.back(); + if (biasShape.size() == 1) + return biasShape[0] == n; + if (biasShape.back() != n) + return false; + for (std::size_t i = 0; i + 1 < biasShape.size(); ++i) { + if (biasShape[i] != 1) + return false; + } + return true; +} + +QuantizationInfo MakeAccumulatorBiasQuantization(const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant) +{ + QuantizationInfo biasQuant; + biasQuant.bitWidth = 32; + biasQuant.isSigned = true; + biasQuant.narrow = false; + biasQuant.scale = inputQuant.scale * weightQuant.scale; + biasQuant.zeroPoint = 0; + biasQuant.rounding = inputQuant.rounding; + biasQuant.overflow = EQuantizationOverflowMode::SAT; + biasQuant.granularity = weightQuant.granularity == EQuantizationGranularity::PerChannel + ? EQuantizationGranularity::PerChannel + : EQuantizationGranularity::PerTensor; + biasQuant.axis = weightQuant.granularity == EQuantizationGranularity::PerChannel ? 0 : -1; + biasQuant.scaleTensor = weightQuant.scaleTensor; + biasQuant.zeroPointTensor = weightQuant.zeroPointTensor; + return biasQuant; +} + +} // namespace SOFIE diff --git a/core/src/RQuantization_DenseLinear.cxx b/core/src/RQuantization_DenseLinear.cxx new file mode 100644 index 0000000..35896f1 --- /dev/null +++ b/core/src/RQuantization_DenseLinear.cxx @@ -0,0 +1,782 @@ +#include "SOFIE/RQuantization_DenseLinear.hxx" +#include "SOFIE/RQuantization_Analysis.hxx" + +#include +#include +#include + +namespace SOFIE { + +namespace { + +constexpr std::size_t kCublasLtInt8Alignment = 16; +constexpr std::size_t kCublasLtMinOptimizedMacs = 1'000'000; +constexpr double kCublasLtPaddingCandidateMaxWorkRatio = 1.50; +constexpr std::size_t kCublasLtMinProfitablePaddedMacs = 100'000; +constexpr double kCublasLtProfitablePaddedMaxWorkRatio = 1.10; +constexpr std::size_t kCublasLtMinProfitablePaddedK = 64; +constexpr std::size_t kCublasLtMinProfitablePaddedN = 64; + +std::size_t RoundUpToMultiple(std::size_t value, std::size_t multiple) +{ + if (multiple == 0 || value == 0) + return value; + return ((value + multiple - 1) / multiple) * multiple; +} + +bool IsAlignedTo(std::size_t value, std::size_t multiple) +{ + return multiple != 0 && (value % multiple) == 0; +} + +bool IsScalarZeroPointZero(const QuantizationInfo &info) +{ + return info.zeroPoint == 0; +} + +} // namespace + +std::vector QuantizeTensorToInt8(const float *data, std::size_t length, const QuantizationInfo &info) +{ + std::vector quantized(length); + for (std::size_t i = 0; i < length; ++i) { + quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); + } + return quantized; +} + +std::vector QuantizeTensorToUInt8(const float *data, std::size_t length, const QuantizationInfo &info) +{ + std::vector quantized(length); + for (std::size_t i = 0; i < length; ++i) { + quantized[i] = static_cast(QuantizeScalarToIntegerGrid(data[i], info)); + } + return quantized; +} + +std::vector QuantizeGemmWeightTensorToInt8(const float *data, std::size_t n, std::size_t k, + const QuantizationInfo &info, + const std::vector &perChannelScale) +{ + std::vector quantized(n * k); + for (std::size_t col = 0; col < n; ++col) { + QuantizationInfo channelInfo = info; + channelInfo.granularity = EQuantizationGranularity::PerTensor; + channelInfo.axis = -1; + channelInfo.scale = static_cast(perChannelScale[col]); + channelInfo.zeroPoint = 0; + for (std::size_t kk = 0; kk < k; ++kk) { + quantized[col * k + kk] = static_cast(QuantizeScalarToIntegerGrid(data[col * k + kk], channelInfo)); + } + } + return quantized; +} + +std::vector QuantizeMatMulWeightTensorToInt8Transposed(const float *data, std::size_t k, std::size_t n, + const QuantizationInfo &info, + const std::vector &perChannelScale) +{ + std::vector quantized(n * k); + for (std::size_t col = 0; col < n; ++col) { + QuantizationInfo channelInfo = info; + if (!perChannelScale.empty()) { + channelInfo.granularity = EQuantizationGranularity::PerTensor; + channelInfo.axis = -1; + channelInfo.scale = static_cast(perChannelScale[col]); + channelInfo.zeroPoint = 0; + } + for (std::size_t kk = 0; kk < k; ++kk) { + quantized[col * k + kk] = static_cast(QuantizeScalarToIntegerGrid(data[kk * n + col], channelInfo)); + } + } + return quantized; +} + +std::vector QuantizeGemmWeightTensorToInt8Padded(const float *data, std::size_t n, std::size_t k, + std::size_t physicalN, std::size_t physicalK, + const QuantizationInfo &info, + const std::vector &perChannelScale) +{ + std::vector quantized(physicalN * physicalK, 0); + for (std::size_t col = 0; col < n; ++col) { + QuantizationInfo channelInfo = info; + if (!perChannelScale.empty()) { + channelInfo.granularity = EQuantizationGranularity::PerTensor; + channelInfo.axis = -1; + channelInfo.scale = static_cast(perChannelScale[col]); + channelInfo.zeroPoint = 0; + } + for (std::size_t kk = 0; kk < k; ++kk) { + quantized[col * physicalK + kk] = static_cast(QuantizeScalarToIntegerGrid(data[col * k + kk], channelInfo)); + } + } + return quantized; +} + +std::vector QuantizeMatMulWeightTensorToInt8TransposedPadded(const float *data, std::size_t k, std::size_t n, + std::size_t physicalK, std::size_t physicalN, + const QuantizationInfo &info, + const std::vector &perChannelScale) +{ + std::vector quantized(physicalN * physicalK, 0); + for (std::size_t col = 0; col < n; ++col) { + QuantizationInfo channelInfo = info; + if (!perChannelScale.empty()) { + channelInfo.granularity = EQuantizationGranularity::PerTensor; + channelInfo.axis = -1; + channelInfo.scale = static_cast(perChannelScale[col]); + channelInfo.zeroPoint = 0; + } + for (std::size_t kk = 0; kk < k; ++kk) { + quantized[col * physicalK + kk] = static_cast(QuantizeScalarToIntegerGrid(data[kk * n + col], channelInfo)); + } + } + return quantized; +} + +bool IsScalarPerTensor(const QuantizationInfo &info) +{ + return info.granularity == EQuantizationGranularity::PerTensor && info.axis == -1; +} + +bool IsPerChannelAxis(const QuantizationInfo &info, int axis) +{ + return info.granularity == EQuantizationGranularity::PerChannel && info.axis == axis; +} + +std::vector DenseLinearQuantizationParameterUnsupportedReasons( + const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant, + const QuantizationInfo &outputQuant, + const std::optional &biasQuant, + int expectedWeightPerChannelAxis, + const std::string &operatorName) +{ + std::vector reasons; + + const bool inputPerTensor = IsScalarPerTensor(inputQuant); + const bool outputPerTensor = IsScalarPerTensor(outputQuant); + const bool weightPerTensor = IsScalarPerTensor(weightQuant); + const bool weightPerChannel = IsPerChannelAxis(weightQuant, expectedWeightPerChannelAxis); + + if (!inputPerTensor) + reasons.push_back(operatorName + " input quantization requires per-tensor scalar parameters"); + if (!outputPerTensor) + reasons.push_back(operatorName + " output quantization requires per-tensor scalar parameters"); + if (!weightPerTensor && !weightPerChannel) { + reasons.push_back(operatorName + " weight quantization supports per-tensor scalar parameters or per-output-channel axis " + + std::to_string(expectedWeightPerChannelAxis)); + } + + if (biasQuant.has_value()) { + const bool biasPerOutputChannel = IsPerChannelAxis(*biasQuant, 0) || + IsPerChannelAxis(*biasQuant, expectedWeightPerChannelAxis); + if (weightPerChannel && !biasPerOutputChannel) { + reasons.push_back(operatorName + " per-channel weight quantization requires per-output-channel bias parameters"); + } else if (weightPerTensor && !IsScalarPerTensor(*biasQuant)) { + reasons.push_back(operatorName + " per-tensor weight quantization requires per-tensor bias parameters"); + } + } + + return reasons; +} + +QuantizedDenseLinearProfileAssessment AssessDenseLinearComputeProfile( + const QuantizationInfo &inputQuant, + const QuantizationInfo &weightQuant, + const QuantizationInfo &outputQuant, + int expectedWeightPerChannelAxis, + const std::string &operatorName) +{ + QuantizedDenseLinearProfileAssessment assessment; + + const bool input8 = inputQuant.bitWidth == 8; + const bool weight8 = weightQuant.bitWidth == 8; + const bool output8 = outputQuant.bitWidth == 8; + if (!input8) + assessment.reasons.push_back(operatorName + " input bit width is not 8"); + if (!weight8) + assessment.reasons.push_back(operatorName + " weight bit width is not 8"); + if (!output8) + assessment.reasons.push_back(operatorName + " output bit width is not 8"); + + const bool inputPerTensor = IsScalarPerTensor(inputQuant); + const bool outputPerTensor = IsScalarPerTensor(outputQuant); + const bool weightPerTensor = IsScalarPerTensor(weightQuant); + const bool weightPerChannel = IsPerChannelAxis(weightQuant, expectedWeightPerChannelAxis); + auto parameterReasons = DenseLinearQuantizationParameterUnsupportedReasons( + inputQuant, weightQuant, outputQuant, std::nullopt, expectedWeightPerChannelAxis, operatorName); + assessment.reasons.insert(assessment.reasons.end(), parameterReasons.begin(), parameterReasons.end()); + + const bool hasAsymmetricInput = !IsScalarZeroPointZero(inputQuant); + const bool hasAsymmetricWeight = !IsScalarZeroPointZero(weightQuant); + const bool hasAsymmetricOutput = !IsScalarZeroPointZero(outputQuant); + if (hasAsymmetricInput) { + assessment.reasons.push_back(operatorName + " input zero point is nonzero; cuBLASLt int8 lowering requires row-sum zero-point correction"); + } + if (hasAsymmetricWeight) { + assessment.reasons.push_back(operatorName + " weight zero point is nonzero; cuBLASLt int8 lowering requires activation-sum/weight-sum zero-point correction"); + } + if (hasAsymmetricOutput) { + assessment.reasons.push_back(operatorName + " output zero point is nonzero; cuBLASLt int8 lowering emits zero-centered int8 output only"); + } + + if (!input8 || !weight8 || !output8 || !inputPerTensor || !outputPerTensor || (!weightPerTensor && !weightPerChannel)) { + assessment.profile = EQuantizedComputeProfile::UnsupportedDenseLinearRank2; + return assessment; + } + + if (hasAsymmetricInput || hasAsymmetricWeight || hasAsymmetricOutput) { + assessment.profile = EQuantizedComputeProfile::AsymmetricZeroPointRank2; + return assessment; + } + + if (inputQuant.isSigned && weightQuant.isSigned && outputQuant.isSigned) { + assessment.profile = weightPerChannel ? EQuantizedComputeProfile::SignedInt8PerTensorActivationPerChannelWeightRank2 + : EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; + assessment.cublasLtOptimizedCandidate = true; + return assessment; + } + + if (!inputQuant.isSigned && weightQuant.isSigned) { + assessment.profile = EQuantizedComputeProfile::UnsignedInt8ActivationSignedInt8WeightRank2; + assessment.reasons.push_back(operatorName + " unsigned activation profile is recognized but the cuBLASLt int8 lowering supports signed int8 carriers only"); + return assessment; + } + + if (!inputQuant.isSigned && !weightQuant.isSigned) { + assessment.profile = EQuantizedComputeProfile::UnsignedInt8SymmetricRank2; + assessment.reasons.push_back(operatorName + " unsigned activation/weight profile is recognized but the cuBLASLt int8 lowering supports signed int8 carriers only"); + return assessment; + } + + assessment.profile = EQuantizedComputeProfile::UnsupportedDenseLinearRank2; + assessment.reasons.push_back(operatorName + " signedness combination is not supported by dense-linear int8 lowering"); + return assessment; +} + +QuantizedDenseLinearShapePolicy MakeCublasLtShapePolicy(std::size_t m, std::size_t k, std::size_t n) +{ + QuantizedDenseLinearShapePolicy policy; + policy.logicalM = m; + policy.logicalK = k; + policy.logicalN = n; + policy.physicalM = RoundUpToMultiple(m, kCublasLtInt8Alignment); + policy.physicalK = RoundUpToMultiple(k, kCublasLtInt8Alignment); + policy.physicalN = RoundUpToMultiple(n, kCublasLtInt8Alignment); + + policy.logicalMacs = m * k * n; + policy.physicalMacs = policy.physicalM * policy.physicalK * policy.physicalN; + policy.minimumOptimizedMacs = kCublasLtMinOptimizedMacs; + policy.belowMinimumWork = policy.logicalMacs < policy.minimumOptimizedMacs; + policy.paddingWorkRatio = policy.logicalMacs > 0 ? static_cast(policy.physicalMacs) / + static_cast(policy.logicalMacs) : 1.0; + + std::ostringstream reason; + reason << "logical M/K/N=" << policy.logicalM << "/" << policy.logicalK << "/" << policy.logicalN + << ", physical M/K/N=" << policy.physicalM << "/" << policy.physicalK << "/" << policy.physicalN + << ", logical MACs=" << policy.logicalMacs + << ", physical MACs=" << policy.physicalMacs + << ", minimum optimized MACs=" << policy.minimumOptimizedMacs + << ", padding work ratio=" << policy.paddingWorkRatio; + + if (IsAlignedTo(m, kCublasLtInt8Alignment) && IsAlignedTo(k, kCublasLtInt8Alignment) && + IsAlignedTo(n, kCublasLtInt8Alignment)) { + if (policy.belowMinimumWork) { + policy.policy = EQuantizedShapePolicy::ExactTooSmall; + policy.reason = "exact cuBLASLt int8 shape below minimum optimized work threshold; " + reason.str(); + } else { + policy.policy = EQuantizedShapePolicy::Exact; + policy.reason = "exact cuBLASLt int8 shape; " + reason.str(); + } + } else if (policy.paddingWorkRatio <= kCublasLtPaddingCandidateMaxWorkRatio) { + policy.policy = EQuantizedShapePolicy::PaddedCandidate; + policy.reason = "padded cuBLASLt candidate; profitability policy selects executable padded lowering; " + reason.str(); + } else { + policy.policy = EQuantizedShapePolicy::Fallback; + policy.reason = "padding too expensive for cuBLASLt candidate; " + reason.str(); + } + return policy; +} + + +bool IsProfitableCublasLtPaddedDenseLinearPolicy(const QuantizedDenseLinearShapePolicy &policy) +{ + return policy.policy == EQuantizedShapePolicy::PaddedCandidate && + policy.logicalMacs >= kCublasLtMinProfitablePaddedMacs && + policy.paddingWorkRatio <= kCublasLtProfitablePaddedMaxWorkRatio && + policy.logicalK >= kCublasLtMinProfitablePaddedK && + policy.logicalN >= kCublasLtMinProfitablePaddedN; +} + +std::string ExplainCublasLtPaddedDenseLinearProfitability(const QuantizedDenseLinearShapePolicy &policy) +{ + std::ostringstream reason; + reason << "padded cuBLASLt profitability requires logical MACs >= " << kCublasLtMinProfitablePaddedMacs + << ", padding work ratio <= " << kCublasLtProfitablePaddedMaxWorkRatio + << ", logical K >= " << kCublasLtMinProfitablePaddedK + << ", and logical N >= " << kCublasLtMinProfitablePaddedN + << "; observed logical MACs=" << policy.logicalMacs + << ", padding work ratio=" << policy.paddingWorkRatio + << ", logical K=" << policy.logicalK + << ", logical N=" << policy.logicalN; + return reason.str(); +} + +namespace { + +std::string JoinCapabilityReasons(const std::vector &reasons) +{ + std::ostringstream out; + for (std::size_t i = 0; i < reasons.size(); ++i) { + if (i != 0) + out << "; "; + out << reasons[i]; + } + return out.str(); +} + +std::size_t DenseLinearLeadingElementCount(const std::vector &shape) +{ + if (shape.size() <= 1) + return 0; + std::size_t count = 1; + for (std::size_t i = 0; i + 1 < shape.size(); ++i) + count *= shape[i]; + return count; +} + +bool DenseLinearShapeMatches(const std::vector &lhs, const std::vector &rhs) +{ + return lhs.size() == rhs.size() && std::equal(lhs.begin(), lhs.end(), rhs.begin()); +} + +std::string DenseLinearShapeToString(const std::vector &shape) +{ + std::ostringstream out; + out << "["; + for (std::size_t i = 0; i < shape.size(); ++i) { + if (i != 0) + out << ","; + out << shape[i]; + } + out << "]"; + return out.str(); +} + +void AddCapabilityReason(std::vector &reasons, std::string reason) +{ + reasons.push_back(std::move(reason)); +} + +} // namespace + +QuantizedMatMulShapeAssessment AssessQuantizedMatMulShape( + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape) +{ + QuantizedMatMulShapeAssessment assessment; + + if (inputShape.size() < 2) + assessment.unsupportedReasons.push_back("MatMul input rank is less than 2"); + if (weightShape.size() < 2) + assessment.unsupportedReasons.push_back("MatMul weight rank is less than 2"); + if (!outputShape.empty() && outputShape.size() < 2) + assessment.unsupportedReasons.push_back("MatMul output rank is less than 2"); + + if (!assessment.unsupportedReasons.empty()) { + assessment.kind = EQuantizedMatMulShapeKind::Unsupported; + assessment.reason = JoinCapabilityReasons(assessment.unsupportedReasons); + return assessment; + } + + const auto inputK = inputShape.back(); + const auto weightK = weightShape[weightShape.size() - 2]; + const auto weightN = weightShape.back(); + + if (inputK == 0 || weightK == 0 || weightN == 0) + assessment.unsupportedReasons.push_back("MatMul K and N dimensions must be nonzero"); + if (inputK != weightK) + assessment.unsupportedReasons.push_back("MatMul input K does not match weight K"); + + std::vector expectedOutput = inputShape; + expectedOutput.back() = weightN; + + if (weightShape.size() == 2) { + if (!outputShape.empty() && !DenseLinearShapeMatches(outputShape, expectedOutput)) { + assessment.unsupportedReasons.push_back( + "MatMul output shape does not match X" + DenseLinearShapeToString(inputShape) + + " @ W" + DenseLinearShapeToString(weightShape) + + " -> Y" + DenseLinearShapeToString(expectedOutput)); + } + if (!assessment.unsupportedReasons.empty()) { + assessment.kind = EQuantizedMatMulShapeKind::Unsupported; + assessment.reason = JoinCapabilityReasons(assessment.unsupportedReasons); + return assessment; + } + + assessment.logicalM = inputShape.size() == 2 ? inputShape[0] : DenseLinearLeadingElementCount(inputShape); + assessment.logicalK = inputK; + assessment.logicalN = weightN; + assessment.flattenedInputShape = { assessment.logicalM, assessment.logicalK }; + assessment.flattenedOutputShape = { assessment.logicalM, assessment.logicalN }; + if (inputShape.size() == 2) { + assessment.kind = EQuantizedMatMulShapeKind::Rank2; + assessment.reason = "rank-2 MatMul shape X[M,K] @ W[K,N] -> Y[M,N]"; + } else { + assessment.kind = EQuantizedMatMulShapeKind::FlattenableProjection; + assessment.reason = "flattenable projection MatMul shape " + DenseLinearShapeToString(inputShape) + + " @ " + DenseLinearShapeToString(weightShape) + + " can be viewed as [prod(prefix),K] @ [K,N]"; + } + return assessment; + } + + if (inputShape.size() >= 3 && weightShape.size() >= 3) { + if (!assessment.unsupportedReasons.empty()) { + assessment.kind = EQuantizedMatMulShapeKind::Unsupported; + assessment.reason = JoinCapabilityReasons(assessment.unsupportedReasons); + return assessment; + } + assessment.logicalM = inputShape[inputShape.size() - 2]; + assessment.logicalK = inputK; + assessment.logicalN = weightN; + assessment.kind = EQuantizedMatMulShapeKind::TrueBatched; + assessment.reason = "true batched MatMul requires strided-batched quantized lowering"; + return assessment; + } + + assessment.kind = EQuantizedMatMulShapeKind::Unsupported; + assessment.unsupportedReasons.push_back("MatMul broadcasted shape family is not a dense projection or true batched MatMul"); + assessment.reason = JoinCapabilityReasons(assessment.unsupportedReasons); + return assessment; +} + +QuantizedDenseLinearCublasLtCapability AssessCublasLtDenseLinearCapability( + const QuantizedDenseLinearOperands &operands) +{ + QuantizedDenseLinearCublasLtCapability capability; + std::vector semanticReasons; + + if (operands.inputShape.size() != 2) + AddCapabilityReason(semanticReasons, operands.operatorName + " input rank is not 2"); + if (operands.weightShape.size() != 2) + AddCapabilityReason(semanticReasons, operands.operatorName + " weight rank is not 2"); + if (!operands.outputShape.empty() && operands.outputShape.size() != 2) + AddCapabilityReason(semanticReasons, operands.operatorName + " output rank is not 2"); + + if (operands.inputShape.size() == 2 && operands.weightShape.size() == 2) { + const auto m = operands.inputShape[0]; + const auto k = operands.inputShape[1]; + const auto n = operands.weightOutputChannelAxis == 0 ? operands.weightShape[0] : operands.weightShape[1]; + const auto weightK = operands.weightOutputChannelAxis == 0 ? operands.weightShape[1] : operands.weightShape[0]; + if (m == 0 || n == 0 || k == 0) + AddCapabilityReason(semanticReasons, operands.operatorName + " M, N, and K must be nonzero"); + if (k != weightK) + AddCapabilityReason(semanticReasons, operands.operatorName + " input K does not match weight K"); + if (!operands.outputShape.empty() && operands.outputShape.size() == 2 && + (operands.outputShape[0] != m || operands.outputShape[1] != n)) { + AddCapabilityReason(semanticReasons, operands.operatorName + " output shape does not match X[M,K] @ W -> Y[M,N]"); + } + if (m != 0 && n != 0 && k != 0 && k == weightK) + capability.shapePolicy = MakeCublasLtShapePolicy(m, k, n); + } + + const auto computeProfile = AssessDenseLinearComputeProfile(operands.inputQuant, operands.weightQuant, + operands.outputQuant, + operands.weightOutputChannelAxis, + operands.operatorName); + semanticReasons.insert(semanticReasons.end(), computeProfile.reasons.begin(), computeProfile.reasons.end()); + + const bool perTensorWeight = IsScalarPerTensor(operands.weightQuant); + const bool perChannelWeight = IsPerChannelAxis(operands.weightQuant, operands.weightOutputChannelAxis); + if (operands.biasQuant.has_value()) { + const bool biasPerOutputChannel = IsPerChannelAxis(*operands.biasQuant, 0) || + IsPerChannelAxis(*operands.biasQuant, operands.weightOutputChannelAxis); + if (perChannelWeight && !biasPerOutputChannel) { + AddCapabilityReason(semanticReasons, operands.operatorName + " per-channel weight requires per-output-channel bias parameters"); + } + if (!perChannelWeight && !IsScalarPerTensor(*operands.biasQuant)) { + AddCapabilityReason(semanticReasons, operands.operatorName + " per-tensor weight requires per-tensor bias parameters"); + } + } + + if (!semanticReasons.empty()) { + capability.shapePolicy.policy = EQuantizedShapePolicy::Unsupported; + capability.shapePolicy.reason = "cuBLASLt semantic requirements are not met"; + capability.profile = computeProfile.profile; + capability.reason = JoinCapabilityReasons(semanticReasons); + capability.tag = "cublaslt_dense_linear_profile_unsupported"; + return capability; + } + + capability.profile = computeProfile.profile; + if (capability.shapePolicy.policy == EQuantizedShapePolicy::Exact) { + capability.optimized = true; + capability.tag = perChannelWeight ? "cublaslt_i8i8_per_channel_weight_rank2_exact" + : "cublaslt_i8i8_symmetric_per_tensor_rank2_exact"; + capability.reason = perChannelWeight ? + "cuBLASLt optimized signed-int8 per-tensor activation/per-channel weight rank-2 exact-shape " + operands.operatorName + "; " + capability.shapePolicy.reason : + "cuBLASLt optimized signed-int8 symmetric per-tensor rank-2 exact-shape " + operands.operatorName + "; " + capability.shapePolicy.reason; + } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::ExactTooSmall) { + capability.tag = perChannelWeight ? "cublaslt_i8i8_per_channel_weight_rank2_exact_too_small" + : "cublaslt_i8i8_symmetric_per_tensor_rank2_exact_too_small"; + capability.reason = "cuBLASLt exact-shape execution is legal but below the minimum optimized work threshold; " + + capability.shapePolicy.reason; + } else if (capability.shapePolicy.policy == EQuantizedShapePolicy::PaddedCandidate) { + capability.tag = perChannelWeight ? "cublaslt_i8i8_per_channel_weight_rank2_padded_candidate" + : "cublaslt_i8i8_symmetric_per_tensor_rank2_padded_candidate"; + capability.reason = "cuBLASLt padded execution is shape-compatible; profitability policy selects executable padded lowering; " + + capability.shapePolicy.reason; + } else { + capability.tag = perChannelWeight ? "cublaslt_i8i8_per_channel_weight_rank2_shape_fallback" + : "cublaslt_i8i8_symmetric_per_tensor_rank2_shape_fallback"; + capability.reason = capability.shapePolicy.reason.empty() ? + "cuBLASLt shape policy is unavailable" : capability.shapePolicy.reason; + } + return capability; +} + +QuantizedLoweringPlan MakeUnsupportedQuantizedMatMulPlan(const QuantizedMatMulRegion ®ion, + EQuantizedBackend backend, + std::string reason, + bool preservesSemantics) +{ + QuantizedLoweringPlan plan; + plan.backend = backend; + plan.status = preservesSemantics ? EQuantizedLoweringStatus::BackendUnsupported + : EQuantizedLoweringStatus::SemanticUnsupported; + plan.reason = std::move(reason); + plan.inputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.weightStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.biasStorage = EQuantizedStorageType::UNDEFINED; + plan.accumulatorStorage = EQuantizedStorageType::UNDEFINED; + plan.outputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.inputCarrierMode = preservesSemantics ? EQuantizedCarrierMode::Float : EQuantizedCarrierMode::UNDEFINED; + plan.outputMode = preservesSemantics ? EQuantizedOutputMode::ExactFakeQuantFloat : EQuantizedOutputMode::UNDEFINED; + plan.computeProfile = preservesSemantics ? EQuantizedComputeProfile::GenericRecognized : EQuantizedComputeProfile::UNDEFINED; + plan.capabilityTag = preservesSemantics ? "matmul_recognized_backend_unsupported" : "matmul_semantic_unsupported"; + plan.consumedOperatorIndices = QuantizedMatMulConsumedOperatorIndices(region); + plan.preservesQuantizationSemantics = preservesSemantics; + plan.isMetadataOnly = preservesSemantics; + plan.suppressesGraphOperators = false; + return plan; +} + + +QuantizedDenseLinearCublasLtCapability SelectExecutableDenseLinearCapability(QuantizedDenseLinearCublasLtCapability capability) +{ + if (capability.shapePolicy.policy == EQuantizedShapePolicy::PaddedCandidate) { + if (IsProfitableCublasLtPaddedDenseLinearPolicy(capability.shapePolicy)) { + capability.shapePolicy.policy = EQuantizedShapePolicy::Padded; + capability.shapePolicy.reason = "padded cuBLASLt int8 execution selected by profitability policy; " + capability.shapePolicy.reason; + capability.optimized = true; + auto pos = capability.tag.find("padded_candidate"); + if (pos != std::string::npos) { + capability.tag.replace(pos, std::string("padded_candidate").size(), "padded"); + } + capability.reason = "cuBLASLt optimized padded signed-int8 rank-2 dense-linear execution; " + capability.shapePolicy.reason; + } else { + capability.shapePolicy.policy = EQuantizedShapePolicy::Fallback; + capability.shapePolicy.reason = "padded cuBLASLt candidate rejected by profitability policy; " + + ExplainCublasLtPaddedDenseLinearProfitability(capability.shapePolicy) + + "; " + capability.shapePolicy.reason; + capability.optimized = false; + auto pos = capability.tag.find("padded_candidate"); + if (pos != std::string::npos) { + capability.tag.replace(pos, std::string("padded_candidate").size(), "padded_unprofitable"); + } + capability.reason = "cuBLASLt padded dense-linear execution is shape-compatible but not expected to beat the baseline; " + + capability.shapePolicy.reason; + } + } + return capability; +} + +QuantizedLoweringPlan MakeMatMulAlpakaTransposedWeightStoragePlan(const QuantizedMatMulRegion ®ion, + const std::string &weightStorageTensor, + const QuantizedDenseLinearShapePolicy &shapePolicy) +{ + QuantizedLoweringPlan plan; + plan.backend = EQuantizedBackend::ALPAKA; + if (QuantizedShapePolicyIsExecutable(shapePolicy.policy)) { + plan.status = EQuantizedLoweringStatus::Optimized; + plan.reason = shapePolicy.policy == EQuantizedShapePolicy::Padded + ? "ALPAKA cuBLASLt MatMul lowering with padded transposed pre-quantized weight storage" + : "ALPAKA cuBLASLt MatMul lowering with transposed pre-quantized weight storage"; + plan.capabilityTag = shapePolicy.policy == EQuantizedShapePolicy::Padded + ? "matmul_cublaslt_i8i8_transposed_weight_rank2_padded" + : "matmul_cublaslt_i8i8_transposed_weight_rank2_exact"; + plan.suppressesGraphOperators = true; + } else { + plan.status = EQuantizedLoweringStatus::BackendUnsupported; + plan.reason = "MatMul transposed pre-quantized weight storage is prepared but the shape is not executable by cuBLASLt int8 lowering: " + + shapePolicy.reason; + plan.capabilityTag = "matmul_transposed_weight_storage_shape_unsupported"; + plan.suppressesGraphOperators = false; + } + plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); + plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); + plan.biasStorage = EQuantizedStorageType::UNDEFINED; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = QuantizedStorageTypeForCarrier(region.outputQuant); + plan.inputCarrierMode = QuantizedCarrierModeForStorage(plan.inputStorage); + plan.outputMode = EQuantizedOutputMode::Quantized; + plan.computeProfile = IsPerChannelAxis(region.weightQuant, 1) + ? EQuantizedComputeProfile::SignedInt8PerTensorActivationPerChannelWeightRank2 + : EQuantizedComputeProfile::SignedInt8SymmetricPerTensorRank2; + plan.shapePolicy = shapePolicy; + plan.weightStorageTensor = weightStorageTensor; + plan.weightLayout = EQuantizedLayout::PlainDevice; + if (IsPerChannelAxis(region.weightQuant, 1)) { + plan.weightScaleMode = EQuantizedParameterMode::PerOutputChannel; + plan.weightScaleTensor = region.weightQuant.scaleTensor; + plan.weightZeroPointTensor = region.weightQuant.zeroPointTensor; + } + plan.consumedOperatorIndices = QuantizedMatMulConsumedOperatorIndices(region); + plan.preservesQuantizationSemantics = true; + plan.isMetadataOnly = false; + return plan; +} + +QuantizedLoweringPlan MakeAvailableQuantizedGemmPlan(const QuantizedGemmRegion ®ion, + EQuantizedBackend backend, + EQuantizedLoweringStatus status, + std::string reason, + std::string capabilityTag) +{ + QuantizedLoweringPlan plan; + plan.backend = backend; + plan.status = status; + plan.reason = std::move(reason); + plan.capabilityTag = std::move(capabilityTag); + plan.consumedOperatorIndices = QuantizedGemmConsumedOperatorIndices(region); + plan.preservesQuantizationSemantics = true; + plan.isMetadataOnly = false; + plan.suppressesGraphOperators = true; + return plan; +} + +QuantizedLoweringPlan MakeCPUPackedWeightBaselinePlan(const QuantizedGemmRegion ®ion, + const std::string &weightStorageTensor) +{ + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::CPU, EQuantizedLoweringStatus::Baseline, + "CPU baseline lowering with packed pre-quantized weight storage", + "cpu_packed_weight_baseline"); + plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); + plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; + plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; + plan.weightStorageTensor = weightStorageTensor; + plan.weightLayout = EQuantizedLayout::PackedCPU; + return plan; +} + +QuantizedLoweringPlan MakeUnsupportedQuantizedGemmPlan(EQuantizedBackend backend, std::string reason, bool preservesSemantics) +{ + QuantizedLoweringPlan plan; + plan.backend = backend; + plan.status = preservesSemantics ? EQuantizedLoweringStatus::BackendUnsupported + : EQuantizedLoweringStatus::SemanticUnsupported; + plan.reason = std::move(reason); + plan.inputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.weightStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.biasStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.accumulatorStorage = EQuantizedStorageType::UNDEFINED; + plan.outputStorage = preservesSemantics ? EQuantizedStorageType::MetadataOnly : EQuantizedStorageType::UNDEFINED; + plan.inputCarrierMode = preservesSemantics ? EQuantizedCarrierMode::Float : EQuantizedCarrierMode::UNDEFINED; + plan.outputMode = preservesSemantics ? EQuantizedOutputMode::ExactFakeQuantFloat : EQuantizedOutputMode::UNDEFINED; + plan.computeProfile = preservesSemantics ? EQuantizedComputeProfile::GenericRecognized : EQuantizedComputeProfile::UNDEFINED; + plan.capabilityTag = preservesSemantics ? "recognized_backend_unsupported" : "semantic_unsupported"; + plan.preservesQuantizationSemantics = preservesSemantics; + plan.isMetadataOnly = preservesSemantics; + plan.suppressesGraphOperators = false; + return plan; +} + +QuantizedLoweringPlan MakeAlpakaFakeQuantPlan(const QuantizedGemmRegion ®ion) +{ + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Baseline, + "Alpaka fake-quant lowering over float carrier tensors", + "alpaka_fake_quant_baseline"); + plan.inputStorage = EQuantizedStorageType::FloatCarrier; + plan.weightStorage = EQuantizedStorageType::FloatCarrier; + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = EQuantizedStorageType::FloatCarrier; + plan.inputCarrierMode = EQuantizedCarrierMode::Float; + plan.outputMode = EQuantizedOutputMode::ExactFakeQuantFloat; + plan.computeProfile = EQuantizedComputeProfile::GenericRecognized; + plan.weightLayout = EQuantizedLayout::Plain; + return plan; +} + + +QuantizedDenseLinearOperands MakeDenseLinearOperands(const QuantizedGemmRegion ®ion, + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape) +{ + QuantizedDenseLinearOperands operands; + operands.inputQuant = region.inputQuant; + operands.weightQuant = region.weightQuant; + operands.outputQuant = region.outputQuant; + operands.biasQuant = region.biasQuant; + operands.inputShape = inputShape; + operands.weightShape = weightShape; + operands.outputShape = outputShape; + operands.weightOutputChannelAxis = 0; + operands.operatorName = "Gemm"; + return operands; +} + +QuantizedDenseLinearOperands MakeDenseLinearOperands(const QuantizedMatMulRegion ®ion, + const std::vector &inputShape, + const std::vector &weightShape, + const std::vector &outputShape) +{ + QuantizedDenseLinearOperands operands; + operands.inputQuant = region.inputQuant; + operands.weightQuant = region.weightQuant; + operands.outputQuant = region.outputQuant; + operands.biasQuant = region.epilogue.biasQuant; + operands.inputShape = inputShape; + operands.weightShape = weightShape; + operands.outputShape = outputShape; + operands.weightOutputChannelAxis = 1; + operands.operatorName = "MatMul"; + return operands; +} + +QuantizedLoweringPlan MakeAlpakaCublasLtCorePlan(const QuantizedGemmRegion ®ion, + const std::string &weightStorageTensor, + const QuantizedDenseLinearCublasLtCapability &capability) +{ + auto plan = MakeAvailableQuantizedGemmPlan(region, EQuantizedBackend::ALPAKA, EQuantizedLoweringStatus::Optimized, + capability.reason, capability.tag); + plan.inputStorage = QuantizedStorageTypeForCarrier(region.inputQuant); + plan.weightStorage = QuantizedStorageTypeForCarrier(region.weightQuant); + plan.biasStorage = EQuantizedStorageType::FloatCarrier; + plan.accumulatorStorage = EQuantizedStorageType::Int32Accumulator; + plan.outputStorage = QuantizedStorageTypeForCarrier(region.outputQuant); + plan.inputCarrierMode = QuantizedCarrierModeForStorage(plan.inputStorage); + plan.outputMode = EQuantizedOutputMode::Quantized; + plan.computeProfile = capability.profile; + plan.shapePolicy = capability.shapePolicy; + plan.weightStorageTensor = weightStorageTensor; + plan.weightLayout = EQuantizedLayout::PlainDevice; + if (IsPerChannelAxis(region.weightQuant, 0)) { + plan.weightScaleMode = EQuantizedParameterMode::PerOutputChannel; + plan.weightScaleTensor = region.weightQuant.scaleTensor; + plan.weightZeroPointTensor = region.weightQuant.zeroPointTensor; + } + return plan; +} + + +} // namespace SOFIE diff --git a/core/src/RQuantization_Parameters.cxx b/core/src/RQuantization_Parameters.cxx new file mode 100644 index 0000000..5379efe --- /dev/null +++ b/core/src/RQuantization_Parameters.cxx @@ -0,0 +1,79 @@ +#include "SOFIE/RQuantization_Parameters.hxx" + +#include +#include + +namespace SOFIE { + +std::vector ValidateIntegralZeroPoints(const std::vector &values, + const std::string &context) +{ + if (values.empty()) + throw std::runtime_error(context + " expected a non-empty zero-point tensor"); + std::vector result; + result.reserve(values.size()); + for (float value : values) { + if (!std::isfinite(value) || std::round(static_cast(value)) != static_cast(value)) + throw std::runtime_error(context + " zero-point values must be finite integers"); + result.push_back(static_cast(std::llround(static_cast(value)))); + } + return result; +} + +int InferQuantizationParameterAxis(const std::vector &tensorShape, + std::size_t parameterCount, + std::optional explicitAxis, + const std::string &context) +{ + if (parameterCount == 1) + return -1; + if (explicitAxis) { + if (*explicitAxis < -static_cast(tensorShape.size()) || + *explicitAxis >= static_cast(tensorShape.size())) + throw std::runtime_error(context + " axis is outside the tensor rank"); + return *explicitAxis < 0 ? *explicitAxis + static_cast(tensorShape.size()) : *explicitAxis; + } + for (std::size_t i = 0; i < tensorShape.size(); ++i) { + if (tensorShape[i] == parameterCount) + return static_cast(i); + } + return -1; +} + +QuantizationInfo MakeValidatedQuantizationInfo(const QuantizationParameterSpec &spec) +{ + if (spec.scales.empty()) + throw std::runtime_error(spec.context + " expected a non-empty scale tensor"); + if (spec.zeroPoints.empty()) + throw std::runtime_error(spec.context + " expected a non-empty zero-point tensor"); + for (double scale : spec.scales) { + if (!(scale > 0.0) || !std::isfinite(scale)) + throw std::runtime_error(spec.context + " scale values must be positive and finite"); + } + if (spec.bitWidth == 0 || spec.bitWidth > 32) + throw std::runtime_error(spec.context + " bit width must be in [1, 32]"); + + const bool vectorScale = spec.scales.size() != 1; + const bool vectorZeroPoint = spec.zeroPoints.size() != 1; + if (vectorScale && vectorZeroPoint && spec.scales.size() != spec.zeroPoints.size()) + throw std::runtime_error(spec.context + " vector scale and zero-point sizes must match"); + const auto parameterCount = vectorScale ? spec.scales.size() : spec.zeroPoints.size(); + + QuantizationInfo info; + info.bitWidth = spec.bitWidth; + info.isSigned = spec.isSigned; + info.narrow = spec.narrow; + info.scale = spec.scales.front(); + info.zeroPoint = spec.zeroPoints.front(); + info.scaleTensor = spec.scaleTensor; + info.zeroPointTensor = spec.zeroPointTensor; + info.rounding = spec.rounding; + info.overflow = spec.overflow; + info.granularity = (vectorScale || vectorZeroPoint) ? EQuantizationGranularity::PerChannel + : EQuantizationGranularity::PerTensor; + info.axis = InferQuantizationParameterAxis(spec.tensorShape, parameterCount, + spec.explicitAxis, spec.context); + return info; +} + +} // namespace SOFIE diff --git a/core/src/RQuantization_Storage.cxx b/core/src/RQuantization_Storage.cxx new file mode 100644 index 0000000..e7d47a4 --- /dev/null +++ b/core/src/RQuantization_Storage.cxx @@ -0,0 +1,149 @@ +#include "SOFIE/RQuantization_Storage.hxx" +#include "SOFIE/RQuantization_DenseLinear.hxx" + +#include +#include + +namespace SOFIE { + +ETensorType TensorTypeForQuantizedStorage(EQuantizedStorageType storage) +{ + switch (storage) { + case EQuantizedStorageType::FloatCarrier: + return ETensorType::FLOAT; + case EQuantizedStorageType::Int8: + return ETensorType::INT8; + case EQuantizedStorageType::UInt8: + return ETensorType::UINT8; + case EQuantizedStorageType::Int32Accumulator: + return ETensorType::INT32; + default: + throw std::runtime_error("SOFIE quantized lowering plan has no physical tensor type for this storage"); + } +} + +QuantizedTensorStorage MakeQuantizedTensorStorage(std::string logicalTensor, + std::string sourceTensor, + std::string storageTensor, + const QuantizationInfo &quantization, + EQuantizedLayout layout, + std::vector shape, + EQuantizedBackend backend) +{ + QuantizedTensorStorage storage; + storage.logicalTensor = std::move(logicalTensor); + storage.sourceTensor = std::move(sourceTensor); + storage.storageTensor = std::move(storageTensor); + storage.storageType = QuantizedStorageTypeForCarrier(quantization); + storage.layout = layout; + storage.quantization = quantization; + storage.shape = std::move(shape); + storage.residentBackend = backend; + return storage; +} + +template +std::vector PackQuantizedGemmWeights(const float *data, std::size_t n, std::size_t k, + std::size_t tileN, const QuantizationInfo &info) +{ + const auto blocks = (n + tileN - 1) / tileN; + std::vector weights(blocks * k * tileN, 0); + for (std::size_t block = 0; block < blocks; ++block) + for (std::size_t kk = 0; kk < k; ++kk) + for (std::size_t lane = 0; lane < tileN; ++lane) { + const auto column = block * tileN + lane; + if (column < n) + weights[(block * k + kk) * tileN + lane] = + static_cast(QuantizeScalarToIntegerGrid(data[column * k + kk], info)); + } + return weights; +} + +std::vector PackQuantizedGemmWeightsInt8(const float *data, std::size_t n, std::size_t k, + std::size_t tileN, const QuantizationInfo &info) +{ + return PackQuantizedGemmWeights(data, n, k, tileN, info); +} + +std::vector PackQuantizedGemmWeightsUInt8(const float *data, std::size_t n, std::size_t k, + std::size_t tileN, const QuantizationInfo &info) +{ + return PackQuantizedGemmWeights(data, n, k, tileN, info); +} + +MaterializedQuantizedWeight MaterializeQuantizedGemmWeight( + const QuantizedGemmRegion ®ion, const QuantizedLoweringPlan &plan, + EQuantizedBackend backend, const float *sourceData, + const std::vector &sourceShape, + const std::vector &perChannelScales) +{ + if (sourceShape.size() != 2) + throw std::runtime_error("SOFIE quantized Gemm storage requires a rank-2 weight tensor"); + const auto n = sourceShape[0]; + const auto k = sourceShape[1]; + MaterializedQuantizedWeight result; + + if (backend == EQuantizedBackend::CPU) { + constexpr std::size_t tileN = 4; + const std::vector shape = {(n + tileN - 1) / tileN, k, tileN}; + result.storage = MakeQuantizedTensorStorage(region.weightTensor, region.weightSourceTensor, + plan.weightStorageTensor, region.weightQuant, + EQuantizedLayout::PackedCPU, shape, backend); + result.buffer = region.weightQuant.isSigned + ? QuantizedWeightBuffer{PackQuantizedGemmWeightsInt8(sourceData, n, k, tileN, + region.weightQuant)} + : QuantizedWeightBuffer{PackQuantizedGemmWeightsUInt8(sourceData, n, k, tileN, + region.weightQuant)}; + return result; + } + + if (backend != EQuantizedBackend::ALPAKA) + throw std::runtime_error("SOFIE quantized Gemm storage received an unsupported backend"); + + std::vector shape = sourceShape; + if (plan.shapePolicy.policy == EQuantizedShapePolicy::Padded) { + shape = {plan.shapePolicy.physicalN, plan.shapePolicy.physicalK}; + result.buffer = QuantizeGemmWeightTensorToInt8Padded(sourceData, n, k, shape[0], shape[1], + region.weightQuant, perChannelScales); + } else if (IsPerChannelAxis(region.weightQuant, 0)) { + result.buffer = QuantizeGemmWeightTensorToInt8(sourceData, n, k, region.weightQuant, perChannelScales); + } else if (region.weightQuant.isSigned) { + result.buffer = QuantizeTensorToInt8(sourceData, n * k, region.weightQuant); + } else { + result.buffer = QuantizeTensorToUInt8(sourceData, n * k, region.weightQuant); + } + result.storage = MakeQuantizedTensorStorage(region.weightTensor, region.weightSourceTensor, + plan.weightStorageTensor, region.weightQuant, + EQuantizedLayout::PlainDevice, shape, backend); + return result; +} + +MaterializedQuantizedWeight MaterializeQuantizedMatMulWeight( + const QuantizedMatMulRegion ®ion, const QuantizedLoweringPlan &plan, + EQuantizedBackend backend, const float *sourceData, + const std::vector &sourceShape, + const std::vector &perChannelScales) +{ + if (backend != EQuantizedBackend::ALPAKA) + throw std::runtime_error("SOFIE quantized MatMul storage currently requires the Alpaka backend"); + if (sourceShape.size() != 2) + throw std::runtime_error("SOFIE quantized MatMul storage requires a rank-2 weight tensor"); + const auto k = sourceShape[0]; + const auto n = sourceShape[1]; + std::vector shape = {n, k}; + MaterializedQuantizedWeight result; + if (plan.shapePolicy.policy == EQuantizedShapePolicy::Padded) { + shape = {plan.shapePolicy.physicalN, plan.shapePolicy.physicalK}; + result.buffer = QuantizeMatMulWeightTensorToInt8TransposedPadded( + sourceData, k, n, shape[1], shape[0], region.weightQuant, perChannelScales); + } else { + result.buffer = QuantizeMatMulWeightTensorToInt8Transposed( + sourceData, k, n, region.weightQuant, perChannelScales); + } + result.storage = MakeQuantizedTensorStorage(region.weightTensor, region.weightSourceTensor, + plan.weightStorageTensor, region.weightQuant, + EQuantizedLayout::PlainDevice, shape, backend); + return result; +} + +} // namespace SOFIE From 5e0cae446bdcd68db06c2f5ffa659a000e95f9a7 Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Thu, 9 Jul 2026 15:35:10 +0200 Subject: [PATCH 5/6] feat: add ONNX Q/DQ support --- core/CMakeLists.txt | 1 + core/inc/SOFIE/OperatorList.hxx | 1 + .../SOFIE/ROperator_ONNXQuantizeLinear.hxx | 334 ++++++++++++++++++ parsers/CMakeLists.txt | 1 + parsers/src/ParseONNXQuantization.cxx | 109 ++++++ parsers/src/RModelParser_ONNX.cxx | 79 +++++ 6 files changed, 525 insertions(+) create mode 100644 core/inc/SOFIE/ROperator_ONNXQuantizeLinear.hxx create mode 100644 parsers/src/ParseONNXQuantization.cxx diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 9103957..00fa96c 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -68,6 +68,7 @@ set(sources_headers SOFIE/ROperator_SubGraph.hxx SOFIE/ROperator_Pad.hxx SOFIE/ROperator_QONNXQuant.hxx + SOFIE/ROperator_ONNXQuantizeLinear.hxx SOFIE/ROperator_Where.hxx SOFIE/ROperator_Einsum.hxx SOFIE/ROperator_Random.hxx diff --git a/core/inc/SOFIE/OperatorList.hxx b/core/inc/SOFIE/OperatorList.hxx index 0be8b6b..4e268e9 100644 --- a/core/inc/SOFIE/OperatorList.hxx +++ b/core/inc/SOFIE/OperatorList.hxx @@ -2,6 +2,7 @@ #include "SOFIE/ROperator_Gemm.hxx" #include "SOFIE/ROperator_QuantizedGemm.hxx" #include "SOFIE/ROperator_QuantizedMatMul.hxx" +#include "SOFIE/ROperator_ONNXQuantizeLinear.hxx" #include "SOFIE/ROperator_Relu.hxx" #include "SOFIE/ROperator_Tanh.hxx" #include "SOFIE/ROperator_LeakyRelu.hxx" diff --git a/core/inc/SOFIE/ROperator_ONNXQuantizeLinear.hxx b/core/inc/SOFIE/ROperator_ONNXQuantizeLinear.hxx new file mode 100644 index 0000000..3a12f3f --- /dev/null +++ b/core/inc/SOFIE/ROperator_ONNXQuantizeLinear.hxx @@ -0,0 +1,334 @@ +#ifndef SOFIE_ROPERATOR_ONNXQUANTIZELINEAR +#define SOFIE_ROPERATOR_ONNXQUANTIZELINEAR + +#include "SOFIE/RModel.hxx" +#include "SOFIE/ROperator.hxx" +#include "SOFIE/RQuantization.hxx" +#include "SOFIE/RQuantization_Parameters.hxx" +#include "SOFIE/SOFIE_common.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +namespace DETAIL { + +inline unsigned BitWidthForONNXQuantizedType(ETensorType type) +{ + switch (type) { + case ETensorType::INT8: + case ETensorType::UINT8: + return 8; + case ETensorType::INT16: + case ETensorType::UINT16: + return 16; + case ETensorType::INT32: + case ETensorType::UINT32: + return 32; + default: + throw std::runtime_error("SOFIE ONNX Q/DQ supports integer zero-point carrier types only"); + } +} + +inline bool IsSignedONNXQuantizedType(ETensorType type) +{ + switch (type) { + case ETensorType::INT8: + case ETensorType::INT16: + case ETensorType::INT32: + return true; + case ETensorType::UINT8: + case ETensorType::UINT16: + case ETensorType::UINT32: + return false; + default: + throw std::runtime_error("SOFIE ONNX Q/DQ supports integer zero-point carrier types only"); + } +} + +inline std::vector GetFloatScaleInitializer(RModel &model, const std::string &tensorName, + const std::string &opName) +{ + auto values = model.GetTensorData(tensorName); + if (values.empty()) { + throw std::runtime_error("SOFIE " + opName + " expected non-empty FLOAT scale initializer " + tensorName); + } + return values; +} + +inline std::vector GetIntegerZeroPointInitializer(RModel &model, const std::string &tensorName, + ETensorType type, const std::string &opName) +{ + if (tensorName.empty()) + return {0}; + + switch (type) { + case ETensorType::INT8: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + case ETensorType::UINT8: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + case ETensorType::INT16: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + case ETensorType::UINT16: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + case ETensorType::INT32: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + case ETensorType::UINT32: { + const auto values = model.GetTensorData(tensorName); + return {values.begin(), values.end()}; + } + default: + throw std::runtime_error("SOFIE " + opName + " zero-point tensor has unsupported integer carrier type"); + } +} + +inline QuantizationInfo MakeONNXQDQInfo(RModel &model, const std::string &scaleTensor, + const std::string &zeroPointTensor, ETensorType carrierType, + const std::vector &tensorShape, int explicitAxis, + const std::string &opName) +{ + if (!model.IsInitializedTensor(scaleTensor)) { + throw std::runtime_error("SOFIE " + opName + " scale must be an initialized tensor"); + } + if (!zeroPointTensor.empty() && !model.IsInitializedTensor(zeroPointTensor)) { + throw std::runtime_error("SOFIE " + opName + " zero-point must be an initialized tensor when provided"); + } + + const auto scales = GetFloatScaleInitializer(model, scaleTensor, opName); + const auto zeroPoints = GetIntegerZeroPointInitializer(model, zeroPointTensor, carrierType, opName); + QuantizationParameterSpec spec; + spec.scales.assign(scales.begin(), scales.end()); + spec.zeroPoints = zeroPoints; + spec.bitWidth = BitWidthForONNXQuantizedType(carrierType); + spec.isSigned = IsSignedONNXQuantizedType(carrierType); + spec.narrow = false; + spec.rounding = EQuantizationRoundingMode::ROUND; + spec.overflow = EQuantizationOverflowMode::SAT; + spec.scaleTensor = scaleTensor; + spec.zeroPointTensor = zeroPointTensor; + spec.tensorShape = tensorShape; + spec.explicitAxis = explicitAxis; + spec.context = "SOFIE " + opName; + return MakeValidatedQuantizationInfo(spec); +} + +inline double ScaleForElement(const QuantizationInfo &info, RModel &model, const std::vector &shape, + std::size_t linearIndex) +{ + if (info.granularity != EQuantizationGranularity::PerChannel || info.axis < 0) + return info.scale; + const auto scales = model.GetTensorData(info.scaleTensor); + if (scales.empty()) + return info.scale; + std::size_t stride = 1; + for (std::size_t i = static_cast(info.axis) + 1; i < shape.size(); ++i) + stride *= shape[i]; + const std::size_t channel = (linearIndex / stride) % scales.size(); + return static_cast(scales[channel]); +} + +inline std::int64_t ZeroPointForElement(const QuantizationInfo &info, RModel &model, const std::vector &shape, + ETensorType carrierType, std::size_t linearIndex) +{ + if (info.zeroPointTensor.empty()) + return info.zeroPoint; + if (info.granularity != EQuantizationGranularity::PerChannel || info.axis < 0) + return info.zeroPoint; + const auto zeroPoints = GetIntegerZeroPointInitializer(model, info.zeroPointTensor, carrierType, "ONNX Q/DQ"); + if (zeroPoints.empty()) + return info.zeroPoint; + std::size_t stride = 1; + for (std::size_t i = static_cast(info.axis) + 1; i < shape.size(); ++i) + stride *= shape[i]; + const std::size_t channel = (linearIndex / stride) % zeroPoints.size(); + return zeroPoints[channel]; +} + +} // namespace DETAIL + +class ROperator_ONNXQuantizeLinear final : public ROperator { +private: + std::string fNX; + std::string fNScale; + std::string fNZeroPoint; + std::string fNY; + ETensorType fOutputType = ETensorType::UNDEFINED; + int fAxis = -1; + std::vector fShape; + QuantizationInfo fInfo; + +public: + ROperator_ONNXQuantizeLinear() = default; + + ROperator_ONNXQuantizeLinear(std::string nameX, std::string nameScale, std::string nameZeroPoint, + std::string nameY, ETensorType outputType, int axis) + : fNX(UTILITY::Clean_name(nameX)), fNScale(UTILITY::Clean_name(nameScale)), + fNZeroPoint(UTILITY::Clean_name(nameZeroPoint)), fNY(UTILITY::Clean_name(nameY)), + fOutputType(outputType), fAxis(axis) + { + fInputTensorNames = fNZeroPoint.empty() ? std::vector{fNX, fNScale} + : std::vector{fNX, fNScale, fNZeroPoint}; + fOutputTensorNames = {fNY}; + } + + bool IsQuantizationBoundary() const override { return true; } + std::string GetQuantizationSourceTensor() const override { return fNX; } + + std::vector TypeInference(std::vector) override { return {fOutputType}; } + std::vector> ShapeInference(std::vector> input) override + { + if (input.empty()) return {}; + return {input.front()}; + } + + void Initialize(RModel &model) override + { + if (!model.CheckIfTensorAlreadyExist(fNX)) + throw std::runtime_error("SOFIE ONNX QuantizeLinear input tensor " + fNX + " is not found in model"); + fShape = model.GetTensorShape(fNX); + fInfo = DETAIL::MakeONNXQDQInfo(model, fNScale, fNZeroPoint, fOutputType, fShape, fAxis, + "ONNX QuantizeLinear"); + model.AddQuantizationInfo(fNY, fInfo); + model.AddIntermediateTensor(fNY, fOutputType, fShape); + model.AddNeededStdLib("cmath"); + model.AddNeededStdLib("cstdint"); + } + + std::string Generate(std::string OpName) override + { + OpName = "op_" + OpName; + if (fInfo.granularity != EQuantizationGranularity::PerTensor) { + throw std::runtime_error("SOFIE ONNX QuantizeLinear literal code generation supports scalar parameters only; vector parameters require fused lowering"); + } + const auto [qMin, qMax] = QuantizedIntegerRange(fInfo); + const auto length = ConvertShapeToLength(fShape); + std::stringstream out; + out << "\n//------ ONNX QUANTIZELINEAR " << OpName << "\n"; + out << SP << "for (size_t id = 0; id < " << length << "; ++id) {\n"; + out << SP << SP << "double q = std::nearbyint((static_cast(tensor_" << fNX << "[id]) / " + << fInfo.scale << ") + " << fInfo.zeroPoint << ");\n"; + out << SP << SP << "q = (q < " << qMin << ") ? " << qMin << " : ((q > " << qMax << ") ? " << qMax << " : q);\n"; + out << SP << SP << "tensor_" << fNY << "[id] = static_cast<" << ConvertTypeToString(fOutputType) << ">(q);\n"; + out << SP << "}\n"; + return out.str(); + } +}; + +class ROperator_ONNXDequantizeLinear final : public ROperator { +private: + std::string fNX; + std::string fNScale; + std::string fNZeroPoint; + std::string fNY; + ETensorType fInputType = ETensorType::UNDEFINED; + int fAxis = -1; + std::vector fShape; + QuantizationInfo fInfo; + + template + std::vector DequantizeInitializedTensor(RModel &model) const + { + const auto values = model.GetTensorData(fNX); + std::vector output(values.size()); + for (std::size_t i = 0; i < values.size(); ++i) { + const double scale = DETAIL::ScaleForElement(fInfo, model, fShape, i); + const auto zeroPoint = DETAIL::ZeroPointForElement(fInfo, model, fShape, fInputType, i); + output[i] = static_cast((static_cast(values[i]) - zeroPoint) * scale); + } + return output; + } + +public: + ROperator_ONNXDequantizeLinear() = default; + + ROperator_ONNXDequantizeLinear(std::string nameX, std::string nameScale, std::string nameZeroPoint, + std::string nameY, ETensorType inputType, int axis) + : fNX(UTILITY::Clean_name(nameX)), fNScale(UTILITY::Clean_name(nameScale)), + fNZeroPoint(UTILITY::Clean_name(nameZeroPoint)), fNY(UTILITY::Clean_name(nameY)), + fInputType(inputType), fAxis(axis) + { + fInputTensorNames = fNZeroPoint.empty() ? std::vector{fNX, fNScale} + : std::vector{fNX, fNScale, fNZeroPoint}; + fOutputTensorNames = {fNY}; + } + + bool IsQuantizationBoundary() const override { return true; } + std::string GetQuantizationSourceTensor() const override { return fNX; } + + std::vector TypeInference(std::vector) override { return {ETensorType::FLOAT}; } + std::vector> ShapeInference(std::vector> input) override + { + if (input.empty()) return {}; + return {input.front()}; + } + + void Initialize(RModel &model) override + { + if (!model.CheckIfTensorAlreadyExist(fNX)) + throw std::runtime_error("SOFIE ONNX DequantizeLinear input tensor " + fNX + " is not found in model"); + fShape = model.GetTensorShape(fNX); + fInfo = DETAIL::MakeONNXQDQInfo(model, fNScale, fNZeroPoint, fInputType, fShape, fAxis, + "ONNX DequantizeLinear"); + model.AddQuantizationInfo(fNY, fInfo); + if (model.IsInitializedTensor(fNX)) { + std::vector values; + switch (fInputType) { + case ETensorType::INT8: values = DequantizeInitializedTensor(model); break; + case ETensorType::UINT8: values = DequantizeInitializedTensor(model); break; + case ETensorType::INT16: values = DequantizeInitializedTensor(model); break; + case ETensorType::UINT16: values = DequantizeInitializedTensor(model); break; + case ETensorType::INT32: values = DequantizeInitializedTensor(model); break; + case ETensorType::UINT32: values = DequantizeInitializedTensor(model); break; + default: + throw std::runtime_error("SOFIE ONNX DequantizeLinear supports initialized integer carriers only"); + } + model.AddConstantTensor(fNY, fShape, values); + } else { + model.AddIntermediateTensor(fNY, ETensorType::FLOAT, fShape); + } + model.AddNeededStdLib("cmath"); + model.AddNeededStdLib("cstdint"); + } + + std::string Generate(std::string OpName) override + { + OpName = "op_" + OpName; + if (fInfo.granularity != EQuantizationGranularity::PerTensor) { + throw std::runtime_error("SOFIE ONNX DequantizeLinear literal code generation supports scalar parameters only; vector parameters require fused lowering"); + } + if (fInputType != ETensorType::INT8 && fInputType != ETensorType::UINT8 && fInputType != ETensorType::INT16 && + fInputType != ETensorType::UINT16 && fInputType != ETensorType::INT32 && fInputType != ETensorType::UINT32) { + throw std::runtime_error("SOFIE ONNX DequantizeLinear literal code generation supports integer carriers only"); + } + const auto length = ConvertShapeToLength(fShape); + std::stringstream out; + out << "\n//------ ONNX DEQUANTIZELINEAR " << OpName << "\n"; + out << SP << "for (size_t id = 0; id < " << length << "; ++id) {\n"; + out << SP << SP << "tensor_" << fNY << "[id] = (static_cast(tensor_" << fNX << "[id]) - " + << fInfo.zeroPoint << ") * " << fInfo.scale << ";\n"; + out << SP << "}\n"; + return out.str(); + } +}; + +} // namespace SOFIE + +#endif // SOFIE_ROPERATOR_ONNXQUANTIZELINEAR diff --git a/parsers/CMakeLists.txt b/parsers/CMakeLists.txt index af87659..248bc2c 100644 --- a/parsers/CMakeLists.txt +++ b/parsers/CMakeLists.txt @@ -42,6 +42,7 @@ set(sources_cxx src/ParseConvTranspose.cxx src/ParseGemm.cxx src/ParseQONNXQuant.cxx + src/ParseONNXQuantization.cxx src/ParseGRU.cxx src/ParseIdentity.cxx src/ParseLeakyRelu.cxx diff --git a/parsers/src/ParseONNXQuantization.cxx b/parsers/src/ParseONNXQuantization.cxx new file mode 100644 index 0000000..a908d48 --- /dev/null +++ b/parsers/src/ParseONNXQuantization.cxx @@ -0,0 +1,109 @@ +#include "SOFIE/RModelParser_ONNX.hxx" +#include "SOFIE/ROperator_ONNXQuantizeLinear.hxx" +#include "onnx_proto3.pb.h" + +#include +#include + +namespace SOFIE { + +namespace { + +int ParseAxisAttribute(const onnx::NodeProto &nodeproto) +{ + int axis = -1; + for (const auto &attribute : nodeproto.attribute()) { + if (attribute.name() == "axis") { + axis = static_cast(attribute.i()); + } else if (attribute.name() == "block_size") { + throw std::runtime_error("TMVA::SOFIE ONNX Q/DQ block quantization is not supported"); + } else if (attribute.name() == "output_dtype") { + throw std::runtime_error("TMVA::SOFIE ONNX Q/DQ output_dtype attribute is not supported; provide an explicit zero-point tensor"); + } else { + throw std::runtime_error("TMVA::SOFIE ONNX Q/DQ unsupported attribute " + attribute.name()); + } + } + return axis; +} + +void CheckQDQInputCount(const onnx::NodeProto &nodeproto, const std::string &opName) +{ + if (nodeproto.input_size() < 2 || nodeproto.input_size() > 3) { + throw std::runtime_error("TMVA::SOFIE ONNX " + opName + " expects two or three inputs: tensor, scale, optional zero_point"); + } + if (nodeproto.output_size() != 1) { + throw std::runtime_error("TMVA::SOFIE ONNX " + opName + " expects one output"); + } +} + +std::string OptionalInputName(const onnx::NodeProto &nodeproto, int index) +{ + if (nodeproto.input_size() <= index) + return {}; + return nodeproto.input(index); +} + +} // namespace + +ParserFuncSignature ParseQuantizeLinear = [](RModelParser_ONNX &parser, const onnx::NodeProto &nodeproto) { + CheckQDQInputCount(nodeproto, "QuantizeLinear"); + + const std::string inputName = nodeproto.input(0); + const std::string scaleName = nodeproto.input(1); + const std::string zeroPointName = OptionalInputName(nodeproto, 2); + const std::string outputName = nodeproto.output(0); + const int axis = ParseAxisAttribute(nodeproto); + + if (!parser.IsRegisteredTensorType(inputName)) { + throw std::runtime_error("TMVA::SOFIE ONNX QuantizeLinear input tensor " + inputName + " has no registered type"); + } + if (!parser.IsRegisteredTensorType(scaleName)) { + throw std::runtime_error("TMVA::SOFIE ONNX QuantizeLinear scale tensor " + scaleName + " has no registered type"); + } + + ETensorType outputType = ETensorType::UINT8; + if (!zeroPointName.empty()) { + if (!parser.IsRegisteredTensorType(zeroPointName)) { + throw std::runtime_error("TMVA::SOFIE ONNX QuantizeLinear zero-point tensor " + zeroPointName + " has no registered type"); + } + outputType = parser.GetTensorType(zeroPointName); + } else if (parser.IsRegisteredTensorType(outputName)) { + outputType = parser.GetTensorType(outputName); + } + + if (!parser.IsRegisteredTensorType(outputName)) { + parser.RegisterTensorType(outputName, outputType); + } + + return std::make_unique(inputName, scaleName, zeroPointName, outputName, + outputType, axis); +}; + +ParserFuncSignature ParseDequantizeLinear = [](RModelParser_ONNX &parser, const onnx::NodeProto &nodeproto) { + CheckQDQInputCount(nodeproto, "DequantizeLinear"); + + const std::string inputName = nodeproto.input(0); + const std::string scaleName = nodeproto.input(1); + const std::string zeroPointName = OptionalInputName(nodeproto, 2); + const std::string outputName = nodeproto.output(0); + const int axis = ParseAxisAttribute(nodeproto); + + if (!parser.IsRegisteredTensorType(inputName)) { + throw std::runtime_error("TMVA::SOFIE ONNX DequantizeLinear input tensor " + inputName + " has no registered type"); + } + if (!parser.IsRegisteredTensorType(scaleName)) { + throw std::runtime_error("TMVA::SOFIE ONNX DequantizeLinear scale tensor " + scaleName + " has no registered type"); + } + if (!zeroPointName.empty() && !parser.IsRegisteredTensorType(zeroPointName)) { + throw std::runtime_error("TMVA::SOFIE ONNX DequantizeLinear zero-point tensor " + zeroPointName + " has no registered type"); + } + + if (!parser.IsRegisteredTensorType(outputName)) { + parser.RegisterTensorType(outputName, ETensorType::FLOAT); + } + + return std::make_unique(inputName, scaleName, zeroPointName, outputName, + parser.GetTensorType(inputName), axis); +}; + +} // namespace SOFIE diff --git a/parsers/src/RModelParser_ONNX.cxx b/parsers/src/RModelParser_ONNX.cxx index 20a8eb5..7ac128e 100644 --- a/parsers/src/RModelParser_ONNX.cxx +++ b/parsers/src/RModelParser_ONNX.cxx @@ -74,6 +74,8 @@ extern ParserFuncSignature ParseSelu; extern ParserFuncSignature ParseSigmoid; extern ParserFuncSignature ParseGemm; extern ParserFuncSignature ParseQONNXQuant; +extern ParserFuncSignature ParseQuantizeLinear; +extern ParserFuncSignature ParseDequantizeLinear; extern ParserFuncSignature ParseRNN; extern ParserFuncSignature ParseLSTM; extern ParserFuncSignature ParsePool; @@ -202,6 +204,46 @@ struct ExtractDataFromTP { } }; template<> +struct ExtractDataFromTP { + static void Copy(onnx::TensorProto * tensor, void * data) { + auto *out = static_cast(data); + for (int i = 0; i < tensor->int32_data_size(); ++i) + out[i] = static_cast(tensor->int32_data(i)); + } +}; +template<> +struct ExtractDataFromTP { + static void Copy(onnx::TensorProto * tensor, void * data) { + auto *out = static_cast(data); + for (int i = 0; i < tensor->int32_data_size(); ++i) + out[i] = static_cast(tensor->int32_data(i)); + } +}; +template<> +struct ExtractDataFromTP { + static void Copy(onnx::TensorProto * tensor, void * data) { + auto *out = static_cast(data); + for (int i = 0; i < tensor->int32_data_size(); ++i) + out[i] = static_cast(tensor->int32_data(i)); + } +}; +template<> +struct ExtractDataFromTP { + static void Copy(onnx::TensorProto * tensor, void * data) { + auto *out = static_cast(data); + for (int i = 0; i < tensor->int32_data_size(); ++i) + out[i] = static_cast(tensor->int32_data(i)); + } +}; +template<> +struct ExtractDataFromTP { + static void Copy(onnx::TensorProto * tensor, void * data) { + auto *out = static_cast(data); + for (int i = 0; i < tensor->uint64_data_size(); ++i) + out[i] = static_cast(tensor->uint64_data(i)); + } +}; +template<> struct ExtractDataFromTP { static void Copy(onnx::TensorProto * tensor, void * data) { tensor->mutable_int64_data()->ExtractSubrange(0, tensor->int64_data_size(), @@ -293,6 +335,8 @@ RModelParser_ONNX::RModelParser_ONNX() noexcept : fOperatorsMapImpl(std::make_un RegisterOperator("Conv", ParseConv); RegisterOperator("ConvTranspose", ParseConvTranspose); RegisterOperator("Gemm", ParseGemm); + RegisterOperator("QuantizeLinear", ParseQuantizeLinear); + RegisterOperator("DequantizeLinear", ParseDequantizeLinear); RegisterOperator("qonnx.custom_op.general", "Quant", ParseQONNXQuant); RegisterOperator("GRU", ParseGRU); RegisterOperator("Identity", ParseIdentity); @@ -713,6 +757,34 @@ void RModelParser_ONNX::ParseONNXGraph(RModel & rmodel, const onnx::GraphProto & allInitializedTensors[input_name] = i; break; } + case ETensorType::INT8: { + std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); + if (verbose) std::cout << "add INT8 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; + rmodel.AddInitializedTensor(input_name, ETensorType::INT8, shape, data); + allInitializedTensors[input_name] = i; + break; + } + case ETensorType::UINT8: { + std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); + if (verbose) std::cout << "add UINT8 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; + rmodel.AddInitializedTensor(input_name, ETensorType::UINT8, shape, data); + allInitializedTensors[input_name] = i; + break; + } + case ETensorType::INT16: { + std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); + if (verbose) std::cout << "add INT16 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; + rmodel.AddInitializedTensor(input_name, ETensorType::INT16, shape, data); + allInitializedTensors[input_name] = i; + break; + } + case ETensorType::UINT16: { + std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); + if (verbose) std::cout << "add UINT16 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; + rmodel.AddInitializedTensor(input_name, ETensorType::UINT16, shape, data); + allInitializedTensors[input_name] = i; + break; + } case ETensorType::INT32: { std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); if (verbose) std::cout << "add INT32 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; @@ -720,6 +792,13 @@ void RModelParser_ONNX::ParseONNXGraph(RModel & rmodel, const onnx::GraphProto & allInitializedTensors[input_name] = i; break; } + case ETensorType::UINT32: { + std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); + if (verbose) std::cout << "add UINT32 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; + rmodel.AddInitializedTensor(input_name, ETensorType::UINT32, shape, data); + allInitializedTensors[input_name] = i; + break; + } case ETensorType::INT64: { std::shared_ptr data = GetInitializedTensorData(tensorproto, fLength); if (verbose) std::cout << "add INT64 initialized tensor " << input_name << " shape " << ConvertShapeToString(shape) << std::endl; From 77058b65c37687f200af80d460917d08a4017379 Mon Sep 17 00:00:00 2001 From: Shaun Lee Date: Thu, 9 Jul 2026 16:15:45 +0200 Subject: [PATCH 6/6] test: QONNX and Q/DQ GEMM/MatMul lowering, external binary quantized weight storage --- core/CMakeLists.txt | 2 + core/inc/SOFIE/RModel_Base.hxx | 4 +- core/inc/SOFIE/RWeightFile.hxx | 98 ++ core/inc/SOFIE/SOFIE_common.hxx | 2 +- core/src/RModel.cxx | 65 +- core/src/RModel_ALPAKA.cxx | 20 + core/src/RWeightFile.cxx | 64 + test/CMakeLists.txt | 8 + .../TestCustomModelsFromONNXForAlpakaCuda.cxx | 186 ++- test/input_models/ONNX_QDQ_QuantGemm.onnx | Bin 0 -> 5212 bytes .../ONNX_QDQ_QuantGemm_PerChannelWeight.onnx | Bin 0 -> 5587 bytes test/input_models/ONNX_QDQ_QuantMatMul.onnx | Bin 0 -> 5164 bytes ...ONNX_QDQ_QuantMatMul_PerChannelWeight.onnx | Bin 0 -> 5539 bytes test/input_models/QONNX_QuantGemm.onnx | Bin 66532 -> 10665 bytes test/input_models/QONNX_QuantGemm_NoBias.onnx | Bin 0 -> 17896 bytes test/input_models/QONNX_QuantMatMul.onnx | Bin 0 -> 9907 bytes test/input_models/QONNX_QuantMatMul_Add.onnx | Bin 0 -> 18256 bytes .../QONNX_QuantMatMul_Padded.onnx | Bin 0 -> 22609 bytes .../references/QONNX_QuantGemm.ref.hxx | 1032 ---------------- .../references/QONNX_QuantGemm_input.ref.hxx | 1035 ----------------- 20 files changed, 416 insertions(+), 2100 deletions(-) create mode 100644 core/inc/SOFIE/RWeightFile.hxx create mode 100644 core/src/RWeightFile.cxx create mode 100644 test/input_models/ONNX_QDQ_QuantGemm.onnx create mode 100644 test/input_models/ONNX_QDQ_QuantGemm_PerChannelWeight.onnx create mode 100644 test/input_models/ONNX_QDQ_QuantMatMul.onnx create mode 100644 test/input_models/ONNX_QDQ_QuantMatMul_PerChannelWeight.onnx create mode 100644 test/input_models/QONNX_QuantGemm_NoBias.onnx create mode 100644 test/input_models/QONNX_QuantMatMul.onnx create mode 100644 test/input_models/QONNX_QuantMatMul_Add.onnx create mode 100644 test/input_models/QONNX_QuantMatMul_Padded.onnx delete mode 100644 test/input_models/references/QONNX_QuantGemm.ref.hxx delete mode 100644 test/input_models/references/QONNX_QuantGemm_input.ref.hxx diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 00fa96c..a665c0a 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -19,6 +19,7 @@ set(sources_headers SOFIE/RModel.hxx SOFIE/RModelProfiler.hxx SOFIE/RModelProfilerGPU.hxx + SOFIE/RWeightFile.hxx SOFIE/ROperator.hxx SOFIE/ROperator_BasicUnary.hxx SOFIE/ROperator_BasicBinary.hxx @@ -91,6 +92,7 @@ set(sources_cxx src/RModel_Base.cxx src/RModel.cxx src/RModel_Quantization.cxx + src/RWeightFile.cxx src/RQuantization_Analysis.cxx src/RQuantization_DenseLinear.cxx src/RQuantization_Parameters.cxx diff --git a/core/inc/SOFIE/RModel_Base.hxx b/core/inc/SOFIE/RModel_Base.hxx index b598652..179377d 100644 --- a/core/inc/SOFIE/RModel_Base.hxx +++ b/core/inc/SOFIE/RModel_Base.hxx @@ -28,6 +28,8 @@ enum class Options { kGNN = 0x8, kGNNComponent = 0x10, kProfile = 0x20, + kBinaryWeightFile = 0x40, + kTextWeightFile = 0x80, }; // Optimization levels inspired by ONNXRuntime. @@ -39,7 +41,7 @@ enum class OptimizationLevel { kExtended = 0x1, }; -enum class WeightFileType { None, RootBinary, Text }; +enum class WeightFileType { None, RootBinary, Text, Binary }; inline std::underlying_type_t operator|(Options opA, Options opB) { diff --git a/core/inc/SOFIE/RWeightFile.hxx b/core/inc/SOFIE/RWeightFile.hxx new file mode 100644 index 0000000..5202cc0 --- /dev/null +++ b/core/inc/SOFIE/RWeightFile.hxx @@ -0,0 +1,98 @@ +#ifndef SOFIE_RWEIGHTFILE +#define SOFIE_RWEIGHTFILE + +#include "SOFIE/SOFIE_common.hxx" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SOFIE { + +inline constexpr char kBinaryWeightFileMagic[8] = {'S', 'O', 'F', 'I', 'E', 'W', 'B', '1'}; +inline constexpr std::uint32_t kBinaryWeightFileVersion = 1; +inline constexpr std::uint32_t kBinaryWeightFileEndianMarker = 0x01020304u; + +using BinaryWeightTensorEntry = std::pair; + +bool IsBinaryWeightTensorType(ETensorType type); + +std::vector +CollectBinaryWeightTensors(const std::unordered_map &tensors); + +long WriteBinaryWeightFile(std::ostream &stream, + const std::unordered_map &tensors); + +template +void ReadBinaryWeightValue(std::istream &stream, T &value) +{ + static_assert(std::is_trivially_copyable_v); + stream.read(reinterpret_cast(&value), sizeof(T)); + if (!stream) + throw std::runtime_error("sofie binary weight file is truncated"); +} + +inline void ReadBinaryWeightFileHeader(std::istream &stream, std::uint64_t expectedTensorCount) +{ + char magic[8]{}; + stream.read(magic, sizeof(magic)); + std::uint32_t version = 0; + std::uint32_t endianMarker = 0; + std::uint64_t tensorCount = 0; + ReadBinaryWeightValue(stream, version); + ReadBinaryWeightValue(stream, endianMarker); + ReadBinaryWeightValue(stream, tensorCount); + if (!std::equal(std::begin(magic), std::end(magic), std::begin(kBinaryWeightFileMagic))) + throw std::runtime_error("sofie binary weight file has an invalid magic header"); + if (version != kBinaryWeightFileVersion) + throw std::runtime_error("sofie binary weight file has an unsupported version"); + if (endianMarker != kBinaryWeightFileEndianMarker) + throw std::runtime_error("sofie binary weight file uses an incompatible byte order"); + if (tensorCount != expectedTensorCount) + throw std::runtime_error("sofie binary weight file tensor count does not match the generated model"); +} + +template +void ReadTensorFromBinaryStream(std::istream &stream, T &target, + const std::string &expectedName, ETensorType expectedType, + const std::vector &expectedShape) +{ + std::uint32_t nameLength = 0; + ReadBinaryWeightValue(stream, nameLength); + std::string name(nameLength, 0); + stream.read(name.data(), name.size()); + std::uint32_t type = 0; + std::uint32_t rank = 0; + ReadBinaryWeightValue(stream, type); + ReadBinaryWeightValue(stream, rank); + std::vector shape(rank); + for (auto &dim : shape) { + std::uint64_t value = 0; + ReadBinaryWeightValue(stream, value); + dim = static_cast(value); + } + std::uint64_t elementCount = 0; + std::uint64_t byteCount = 0; + ReadBinaryWeightValue(stream, elementCount); + ReadBinaryWeightValue(stream, byteCount); + std::size_t expectedElementCount = 1; + for (auto dim : expectedShape) + expectedElementCount *= dim; + using Element = std::remove_cv_t>; + if (name != expectedName || type != static_cast(expectedType) || shape != expectedShape || + elementCount != expectedElementCount || byteCount != expectedElementCount * sizeof(Element)) + throw std::runtime_error("sofie binary weight tensor metadata does not match generated tensor " + expectedName); + stream.read(reinterpret_cast(&target[0]), static_cast(byteCount)); + if (!stream) + throw std::runtime_error("sofie binary weight data is truncated for tensor " + expectedName); +} + +} // namespace SOFIE + +#endif // SOFIE_RWEIGHTFILE diff --git a/core/inc/SOFIE/SOFIE_common.hxx b/core/inc/SOFIE/SOFIE_common.hxx index a2d533f..f5d1332 100644 --- a/core/inc/SOFIE/SOFIE_common.hxx +++ b/core/inc/SOFIE/SOFIE_common.hxx @@ -303,7 +303,7 @@ public: void SetNotWritable() { fIsNotWritable = true;} // set writable initialized tensors - i.e. tensor that must be written in a file void SetWritable() { fIsNotWritable = false;} - // set as constant (needed for non-float initialized tensors) + // embed an initialized tensor in generated code instead of the weight file void SetConstant() { fConstant = true;} template diff --git a/core/src/RModel.cxx b/core/src/RModel.cxx index 443669f..f41728c 100644 --- a/core/src/RModel.cxx +++ b/core/src/RModel.cxx @@ -12,6 +12,7 @@ #include "SOFIE/RModel.hxx" #include "SOFIE/RModelProfiler.hxx" +#include "SOFIE/RWeightFile.hxx" #include "SOFIE/SOFIE_common.hxx" namespace SOFIE { @@ -1375,6 +1376,9 @@ void RModel::GenerateSessionCode() if (fWeightFile == WeightFileType::RootBinary) { fileName += ".root"; } + if (fWeightFile == WeightFileType::Binary) { + fileName += ".bin"; + } fGC += sessionName + "(std::string filename =\"" + fileName + "\""; } else { // no need to pass weight file since it is not used @@ -1525,6 +1529,18 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, lo fUseWeightFile = true; fWeightFile = WeightFileType::RootBinary; } + if (static_cast>(Options::kBinaryWeightFile) & options) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Binary; + } + if (static_cast>(Options::kTextWeightFile) & options) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Text; + } + const auto explicitWeightPolicy = static_cast>(Options::kNoWeightFile) | + static_cast>(Options::kRootBinaryWeightFile) | + static_cast>(Options::kBinaryWeightFile) | + static_cast>(Options::kTextWeightFile); if (fUseWeightFile && !fUseSession) { throw std::runtime_error( "sofie: RModel::Generate: cannot use a separate weight file without generating a Session class"); @@ -1540,6 +1556,12 @@ void RModel::Generate(std::underlying_type_t options, int batchSize, lo // initialize the model including all operators and sub-graphs Initialize(batchSize, verbose); + if ((options & explicitWeightPolicy) == 0 && !fQuantizationState.tensorInfos.empty()) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Binary; + } + if (fWeightFile == WeightFileType::Binary) + AddNeededCustomHeader("SOFIE/RWeightFile.hxx"); BuildLoweredOperatorView(EQuantizedBackend::CPU); AddQuantizedGeneratedHeaders(EQuantizedBackend::CPU); @@ -1613,6 +1635,29 @@ void RModel::ReadInitializedTensorsFromFile(long pos) { fGC += " f.close();\n"; } + if (fWeightFile == WeightFileType::Binary) { + if (!fUseWeightFile) return; + if (fIsGNNComponent || pos != 0) + throw std::runtime_error("SOFIE binary weight files do not support appended GNN component offsets"); + + const auto binaryWeights = CollectBinaryWeightTensors(fInitializedTensors); + fGC += " std::ifstream f(filename, std::ios::binary);\n"; + fGC += " if (!f.is_open()) throw std::runtime_error(\"sofie failed to open binary weight file \" + filename);\n"; + fGC += " SOFIE::ReadBinaryWeightFileHeader(f, " + std::to_string(binaryWeights.size()) + ");\n"; + for (const auto &[name, tensor] : binaryWeights) { + const auto type = tensor->type(); + if (!IsBinaryWeightTensorType(type)) + throw std::runtime_error("sofie tensor " + name + " has a type unsupported by binary weights"); + std::string shape = "{"; + for (auto dim : tensor->shape()) shape += std::to_string(dim) + ","; + shape += "}"; + fGC += " SOFIE::ReadTensorFromBinaryStream(f, tensor_" + name + ", \"tensor_" + name + + "\", static_cast(" + std::to_string(static_cast(type)) + + "), std::vector" + shape + ");\n"; + } + fGC += " f.close();\n"; + } + // generate the code to read initialized tensors from a ROOT data file if(fWeightFile == WeightFileType::RootBinary) { #ifdef SOFIE_SUPPORT_ROOT_BINARY @@ -1666,6 +1711,9 @@ long RModel::WriteInitializedTensorsToFile(std::string filename) { case WeightFileType::Text: fileExtension = ".dat"; break; + case WeightFileType::Binary: + fileExtension = ".bin"; + break; } // If filename is empty, use the model name as the base filename @@ -1674,7 +1722,16 @@ long RModel::WriteInitializedTensorsToFile(std::string filename) { } // Write the initialized tensors to the file - if (fWeightFile == WeightFileType::RootBinary) { + if (fWeightFile == WeightFileType::Binary) { + if (fIsGNNComponent || fIsGNN) + throw std::runtime_error("SOFIE binary weight files do not support appended GNN components"); + std::ofstream f(filename, std::ios::binary | std::ios::trunc); + if (!f.is_open()) + throw std::runtime_error("sofie failed to open binary weight file " + filename); + const auto position = WriteBinaryWeightFile(f, fInitializedTensors); + f.close(); + return position; + } else if (fWeightFile == WeightFileType::RootBinary) { #ifdef SOFIE_SUPPORT_ROOT_BINARY if(fIsGNNComponent || fIsGNN) { throw std::runtime_error("SOFIE-GNN yet not supports writing to a ROOT file."); @@ -2034,9 +2091,13 @@ void RModel::OutputGenerated(std::string filename, bool append) { filename = filename.erase(pos, 4); filename += ".root"; } + if (fWeightFile == WeightFileType::Binary) + filename.replace(pos, 4, ".bin"); } else { filename = fName; - filename += fWeightFile == WeightFileType::Text ? ".dat" : ".root"; + if (fWeightFile == WeightFileType::Text) filename += ".dat"; + else if (fWeightFile == WeightFileType::RootBinary) filename += ".root"; + else if (fWeightFile == WeightFileType::Binary) filename += ".bin"; } WriteInitializedTensorsToFile(filename); } diff --git a/core/src/RModel_ALPAKA.cxx b/core/src/RModel_ALPAKA.cxx index b866238..c9c0c2b 100644 --- a/core/src/RModel_ALPAKA.cxx +++ b/core/src/RModel_ALPAKA.cxx @@ -737,6 +737,8 @@ void RModel::GenerateSessionCode_GPU_ALPAKA() { fileName += ".dat"; if (fWeightFile == WeightFileType::RootBinary) fileName += ".root"; + if (fWeightFile == WeightFileType::Binary) + fileName += ".bin"; fGC += sessionName + "(std::string filename =\"" + fileName + "\""; } else { @@ -850,6 +852,18 @@ void RModel::GenerateGPU_ALPAKA(std::underlying_type_t options, int bat fUseWeightFile = true; fWeightFile = WeightFileType::RootBinary; } + if (static_cast>(Options::kBinaryWeightFile) & options) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Binary; + } + if (static_cast>(Options::kTextWeightFile) & options) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Text; + } + const auto explicitWeightPolicy = static_cast>(Options::kNoWeightFile) | + static_cast>(Options::kRootBinaryWeightFile) | + static_cast>(Options::kBinaryWeightFile) | + static_cast>(Options::kTextWeightFile); if (fUseWeightFile && !fUseSession) { throw std::runtime_error( "sofie: RModel::Generate: cannot use a separate weight file without generating a Session class"); @@ -860,6 +874,12 @@ void RModel::GenerateGPU_ALPAKA(std::underlying_type_t options, int bat throw std::runtime_error("SOFIE GPU does not yet supports GNN Inference."); Initialize(batchSize, verbose); + if ((options & explicitWeightPolicy) == 0 && !fQuantizationState.tensorInfos.empty()) { + fUseWeightFile = true; + fWeightFile = WeightFileType::Binary; + } + if (fWeightFile == WeightFileType::Binary) + AddNeededCustomHeader("SOFIE/RWeightFile.hxx"); BuildLoweredOperatorView(EQuantizedBackend::ALPAKA); AddQuantizedGeneratedHeaders(EQuantizedBackend::ALPAKA); FuseGemmActivations_GPU(); // must run before elementwise fusion (redirects tensors) diff --git a/core/src/RWeightFile.cxx b/core/src/RWeightFile.cxx new file mode 100644 index 0000000..4bd5574 --- /dev/null +++ b/core/src/RWeightFile.cxx @@ -0,0 +1,64 @@ +#include "SOFIE/RWeightFile.hxx" + +#include +#include + +namespace SOFIE { + +bool IsBinaryWeightTensorType(ETensorType type) +{ + return type == ETensorType::FLOAT || type == ETensorType::INT8 || type == ETensorType::UINT8 || + type == ETensorType::INT32 || type == ETensorType::INT64; +} + +std::vector +CollectBinaryWeightTensors(const std::unordered_map &tensors) +{ + std::vector result; + result.reserve(tensors.size()); + for (const auto &[name, tensor] : tensors) { + if (tensor.IsWeightTensor()) + result.emplace_back(name, &tensor); + } + std::sort(result.begin(), result.end(), [](const auto &lhs, const auto &rhs) { return lhs.first < rhs.first; }); + return result; +} + +long WriteBinaryWeightFile(std::ostream &stream, + const std::unordered_map &tensors) +{ + auto writeValue = [&stream](const auto &value) { + stream.write(reinterpret_cast(&value), sizeof(value)); + }; + const auto weights = CollectBinaryWeightTensors(tensors); + stream.write(kBinaryWeightFileMagic, sizeof(kBinaryWeightFileMagic)); + writeValue(kBinaryWeightFileVersion); + writeValue(kBinaryWeightFileEndianMarker); + writeValue(static_cast(weights.size())); + + for (const auto &[sourceName, tensor] : weights) { + const auto type = tensor->type(); + if (!IsBinaryWeightTensorType(type)) + throw std::runtime_error("sofie tensor " + sourceName + " has a type unsupported by binary weights"); + const std::string name = "tensor_" + sourceName; + const auto nameLength = static_cast(name.size()); + const auto typeCode = static_cast(type); + const auto rank = static_cast(tensor->shape().size()); + const auto elementCount = static_cast(ConvertShapeToLength(tensor->shape())); + const auto byteCount = elementCount * GetTypeSize(type); + writeValue(nameLength); + stream.write(name.data(), name.size()); + writeValue(typeCode); + writeValue(rank); + for (auto dim : tensor->shape()) + writeValue(static_cast(dim)); + writeValue(elementCount); + writeValue(byteCount); + stream.write(static_cast(tensor->sharedptr().get()), static_cast(byteCount)); + if (!stream) + throw std::runtime_error("sofie failed to write binary tensor " + name); + } + return static_cast(stream.tellp()); +} + +} // namespace SOFIE diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 72fd8b9..56d24b6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -103,6 +103,14 @@ if (ENABLE_ALPAKA_TESTS) FIXTURES_SETUP sofie-compile-models-onnx-alpaka ) + ROOTTEST_ADD_TEST( + SofieQuantizedStorageExternalized_ONNX_Alpaka + COMMAND ${CMAKE_COMMAND} + -DGENERATED_DIR=${CMAKE_CURRENT_BINARY_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/CheckQuantizedStorageExternalized.cmake + FIXTURES_REQUIRED sofie-compile-models-onnx-alpaka + ) + ########################################################################## # CUDA backend ########################################################################## diff --git a/test/TestCustomModelsFromONNXForAlpakaCuda.cxx b/test/TestCustomModelsFromONNXForAlpakaCuda.cxx index e3ca92a..30ae974 100644 --- a/test/TestCustomModelsFromONNXForAlpakaCuda.cxx +++ b/test/TestCustomModelsFromONNXForAlpakaCuda.cxx @@ -1,5 +1,12 @@ +#include +#include #include #include +#include +#include +#include + +#include "SOFIE/RWeightFile.hxx" // ── Trilu ────────────────────────────────────────────────────────────────── #include "Trilu_upper_FromONNX_GPU_ALPAKA.hxx" @@ -44,9 +51,15 @@ #include "Linear_64_FromONNX_GPU_ALPAKA.hxx" #include "input_models/references/Linear_64.ref.hxx" -#include "QONNX_QuantGemm_FromONNX_GPU_ALPAKA.hxx" -#include "input_models/references/QONNX_QuantGemm_input.ref.hxx" -#include "input_models/references/QONNX_QuantGemm.ref.hxx" +#include "QONNX_QuantGemm_Binary_FromONNX_GPU_ALPAKA.hxx" +#include "QONNX_QuantGemm_NoBias_FromONNX_GPU_ALPAKA.hxx" +#include "QONNX_QuantMatMul_FromONNX_GPU_ALPAKA.hxx" +#include "QONNX_QuantMatMul_Padded_FromONNX_GPU_ALPAKA.hxx" +#include "QONNX_QuantMatMul_Add_FromONNX_GPU_ALPAKA.hxx" +#include "ONNX_QDQ_QuantGemm_FromONNX_GPU_ALPAKA.hxx" +#include "ONNX_QDQ_QuantMatMul_FromONNX_GPU_ALPAKA.hxx" +#include "ONNX_QDQ_QuantGemm_PerChannelWeight_FromONNX_GPU_ALPAKA.hxx" +#include "ONNX_QDQ_QuantMatMul_PerChannelWeight_FromONNX_GPU_ALPAKA.hxx" #include "AddBroadcast1_FromONNX_GPU_ALPAKA.hxx" #include "input_models/references/AddBroadcast1.ref.hxx" @@ -193,6 +206,57 @@ using Idx = std::size_t; using Dim = alpaka::DimInt<1>; using Ext1D = alpaka::Vec; +struct QuantizedLinearTest { + Idx m; + Idx k; + Idx n; + bool matMul; + bool hasBias; + bool perChannelWeight; +}; + +std::int8_t QuantizedLinearTestInputValue(Idx index) +{ + return static_cast(((index * 5 + index / 7) % 31) - 15); +} + +std::int8_t QuantizedLinearTestWeightValue(Idx index, Idx n) +{ + return static_cast(((index * 3 + index / 11 + n) % 29) - 14); +} + +std::vector MakeQuantizedLinearTestInput(const QuantizedLinearTest &test) +{ + std::vector input(test.m * test.k); + for (Idx i = 0; i < input.size(); ++i) + input[i] = QuantizedLinearTestInputValue(i); + return input; +} + +std::vector MakeQuantizedLinearTestExpected(const QuantizedLinearTest &test, + const std::vector &input) +{ + std::vector output(test.m * test.n); + constexpr int scaleNumerators[] = {3, 4, 5, 6}; + for (Idx row = 0; row < test.m; ++row) { + for (Idx column = 0; column < test.n; ++column) { + std::int32_t accumulator = 0; + for (Idx inner = 0; inner < test.k; ++inner) { + const Idx weightIndex = test.matMul ? inner * test.n + column : column * test.k + inner; + accumulator += static_cast(input[row * test.k + inner]) * + static_cast(QuantizedLinearTestWeightValue(weightIndex, test.n)); + } + if (test.hasBias) + accumulator += static_cast((column * 3) % 17) - 8; + const int scaleNumerator = test.perChannelWeight ? scaleNumerators[column % 4] : 4; + const auto quantized = static_cast(std::nearbyint( + static_cast(accumulator) * static_cast(scaleNumerator) / 128.0)); + output[row * test.n + column] = static_cast(std::clamp(quantized, -128L, 127L)); + } + } + return output; +} + class SofieAlpakaTest : public ::testing::Test { protected: // Shared devices and platforms @@ -202,6 +266,41 @@ class SofieAlpakaTest : public ::testing::Test { alpaka::DevCudaRt device; alpaka::Queue queue; + template + void RunQuantizedLinearInt8(const char *weightFile, const QuantizedLinearTest &test) + { + const auto input = MakeQuantizedLinearTestInput(test); + const auto expectedOutput = MakeQuantizedLinearTestExpected(test, input); + const Idx inputSize = input.size(); + const Idx outputSize = expectedOutput.size(); + auto input_h = alpaka::allocBuf(host, Ext1D::all(inputSize)); + std::int8_t *input_ptr = reinterpret_cast(alpaka::getPtrNative(input_h)); + for (Idx i = 0; i < inputSize; ++i) { + input_ptr[i] = input[i]; + } + + auto input_d = alpaka::allocBuf(device, Ext1D::all(inputSize)); + alpaka::memcpy(queue, input_d, input_h); + alpaka::wait(queue); + + auto result_h = alpaka::allocBuf(host, Ext1D::all(outputSize)); + + { + TModel model(weightFile); + auto result = model.infer(input_d); + alpaka::wait(queue); + cudaDeviceSynchronize(); + + alpaka::memcpy(queue, result_h, result); + alpaka::wait(queue); + } + + const auto *res_ptr = reinterpret_cast(alpaka::getPtrNative(result_h)); + for (Idx i = 0; i < outputSize; ++i) { + EXPECT_EQ(static_cast(res_ptr[i]), static_cast(expectedOutput[i])) << "i=" << i; + } + } + SofieAlpakaTest() : hostPlatform{} , host(alpaka::getDevByIdx(hostPlatform, 0u)) @@ -228,39 +327,68 @@ class SofieAlpakaTest : public ::testing::Test { TEST_F(SofieAlpakaTest, QuantizedGemm) { - constexpr Idx inputSize = QONNX_QuantGemm_Input::input_size; - constexpr Idx outputSize = QONNX_QuantGemm_ExpectedOutput::output_size; - static_assert(inputSize == 128 * 128); - static_assert(outputSize == 128 * 128); + // Biased Gemm: Yq = QY(SX * SW_j * sum_k Xq_ik * Wq_jk + SX * SW_j * Bq_j). + RunQuantizedLinearInt8>( + "QONNX_QuantGemm_Binary_FromONNX_GPU_ALPAKA.bin", + QuantizedLinearTest{512, 64, 32, false, true, true}); +} - auto input_h = alpaka::allocBuf(host, Ext1D::all(inputSize)); - std::int8_t *input_ptr = reinterpret_cast(alpaka::getPtrNative(input_h)); - for (Idx i = 0; i < inputSize; ++i) { - input_ptr[i] = QONNX_QuantGemm_Input::input[i]; - } +TEST(SofieBinaryWeights, RejectsInvalidHeader) +{ + std::string bytes(24, 0); + std::stringstream stream(bytes, std::ios::in | std::ios::binary); + EXPECT_THROW(SOFIE::ReadBinaryWeightFileHeader(stream, 0), std::runtime_error); +} - auto input_d = alpaka::allocBuf(device, Ext1D::all(inputSize)); - alpaka::memcpy(queue, input_d, input_h); - alpaka::wait(queue); +TEST_F(SofieAlpakaTest, QuantizedGemmFrontendsEquivalent) +{ + // QONNX Quant and standard Q/DQ encode the same no-bias Gemm semantics. + RunQuantizedLinearInt8>( + "QONNX_QuantGemm_NoBias_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, false, false, false}); + RunQuantizedLinearInt8>( + "ONNX_QDQ_QuantGemm_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, false, false, false}); +} - auto result_h = alpaka::allocBuf(host, Ext1D::all(outputSize)); +TEST_F(SofieAlpakaTest, QuantizedMatMulFrontends) +{ + // MatMul uses W as [K,N]: Yq = QY(SX * SW_j * sum_k Xq_ik * Wq_kj). + RunQuantizedLinearInt8>( + "QONNX_QuantMatMul_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{512, 64, 32, true, false, true}); - { - SOFIE_QONNX_QuantGemm::Session model("QONNX_QuantGemm_FromONNX_GPU_ALPAKA.dat"); - auto result = model.infer(input_d); - alpaka::wait(queue); - cudaDeviceSynchronize(); + // Standard Q/DQ example currently uses the smaller shared 256x64 input. + RunQuantizedLinearInt8>( + "ONNX_QDQ_QuantMatMul_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, true, false, false}); +} - alpaka::memcpy(queue, result_h, result); - alpaka::wait(queue); - } +TEST_F(SofieAlpakaTest, QuantizedMatMulPadded) +{ + // Padded MatMul has logical M=511; the backend may pad physical storage but returns logical Y. + RunQuantizedLinearInt8>( + "QONNX_QuantMatMul_Padded_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{511, 64, 80, true, false, true}); +} - const auto *res_ptr = reinterpret_cast(alpaka::getPtrNative(result_h)); - const auto *correct = QONNX_QuantGemm_ExpectedOutput::output; +TEST_F(SofieAlpakaTest, QuantizedMatMulAdd) +{ + // Projection bias: Yq = QY(SX * SW_j * sum_k Xq_ik * Wq_kj + bias_j). + RunQuantizedLinearInt8>( + "QONNX_QuantMatMul_Add_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, true, true, false}); +} - for (Idx i = 0; i < outputSize; ++i) { - EXPECT_EQ(static_cast(res_ptr[i]), static_cast(correct[i])) << "i=" << i; - } +TEST_F(SofieAlpakaTest, StandardQDQPerChannelDenseLinear) +{ + // Gemm uses output-channel axis 0; MatMul uses output-channel axis 1. + RunQuantizedLinearInt8>( + "ONNX_QDQ_QuantGemm_PerChannelWeight_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, false, false, true}); + RunQuantizedLinearInt8>( + "ONNX_QDQ_QuantMatMul_PerChannelWeight_FromONNX_GPU_ALPAKA.dat", + QuantizedLinearTest{256, 64, 64, true, false, true}); } TEST_F(SofieAlpakaTest, Linear64) diff --git a/test/input_models/ONNX_QDQ_QuantGemm.onnx b/test/input_models/ONNX_QDQ_QuantGemm.onnx new file mode 100644 index 0000000000000000000000000000000000000000..41aef07ac005412540bb423b6ed2b588821df9dd GIT binary patch literal 5212 zcmeHLTWb?R6yC|E>2z}GW70G8;Saz9}t%!55X_QZ|_G=7nRhVgs7@f1_LWm>d~vmW~pop%2$9!<3nTI z;bNN{9XVk?Vky@ltBS(~v-!8|YEPpH_pr0?-ZPBB+9?2b1?6{0l#a`&5K%Yiy{Q?e zAZN$JRt>Nra#7YhDoGSo(W)9{(zTQ*t?2X}$!uj?Ra=%55R?aW}Fp<<<-7=BX33I6v=9fYz z%oA5vw}~r|Ph1tIIE*sqFbbK&C}a+!kX2ONTt&srRaD$Oadm!?xVpSbT-|`gm01u8 zN@S6sL>383%l~666a|Wx_s)v>GH`pOuv2d1Jxg|T;KHCC)a!3 zI%JR2twVQSy>U@HEWPiO?IGR=v!ycL2Q!gw9lCYsQ-^#9&h(*8o-F>)nb&>*fdyJw zQfD)77{+URhAGU}2*I-n47(=g4AC@v(%_X{7n5QxCgm_Pv$AZ*@T?K$gU$L_Q@j(N zMrCN?DP+Py4&~tb&SNw}jEoo^JQ+jy2rH0r#jM70KowMi07DMOe+Q3-c$*O~tvoWU zGMzzH!K*opS927VhVv2a6S{?p9_j5;zZ3akfY)Nyt-;xv24l}&H0}~(^PIYwA7*zN z$}5Ek9AR^5WtC1M@EPY;m-$T~ZY%`Dj->CfV6Esw1WB+PdVbKx)fl=N2Ej3B{pkH6 fW)Dv;JX?6axU@oR!+acuOtHAKDmB@MnQHz4xSpO?oE=s4!G?&{5hhI1@wg~J(?IU(tZQty|jJy&OdD$D%K++UNw2OgTWE)q}(@?bz zoL)gW0txsD!0}|^4Y5(Wh4=SOukX|O7d2WUlNsjJ=k!~jffLqVr)}L*l);lD0Q707 zZeu^{XJN`GmfL&EW%Zz2F94Q>1&5=^uy03Bpjx6rZ3_<`Wdke1bwbn#6_7iX|$uVu=Rzpr&XN zC($J0^2vpZ%O~!z`0f*b(D=_Qo^SE9PdxAAtwZc_@zxRVyz#+Bp0MI`pV%IPb1;2W z2IpXEinoq<>xkbv#655vzqE-ri+}UZtKZQ;4%CN{PeWA{<*`0X2a@$M{L=u6{v|;< z0;rEroW>xbD7AHpNN78v?JI5nS6_e2@k+p{BBax((Te`nM=$VY8evClod)ZAk+0J@ zN1_Y$t{{;fnt@p@iKbvu8zdlIz3q88hMC5kcJJSS&$t&D3PrtcB5KnxqXDEbG1hkP zjZm<$D0LMoxh$E5d5$biiY!e@DUs@vJ|=CLwsEgXoR05!E_xdP%@Si@B?>0jN*zoL zfy#`gKwl-n5lh!;sw-p?f)6OMvdk`AevyNkM`ai@MVtl@8M==iQf-DcnIHxIni zFruV+NPI$_FrY8LKY>p$24sU}dqTX}P%TguBr(bYUC>B?(rQsj0h6+EryrE3!n8d++x4 zKG>a|jx8LLy7A3I+i1ILwvw8)ht}`EH%4I=%ICi>uXAQ`uJq$4Gv#N{d}Va;hnb}B z>Xw`o{w@E0Ykc2AEaT*2ZG>VziC}vKhm{nFBuCn5Al@*64p+*-; zsLQJ))QypZGK*q@l37epGK&dH=5!L*Dl49-%8Dl%*MphTNsQ7-=<=xtr^_ebF#Yz) z4^)4=a(&ZlpIq;C>ySN8w+`KT^~Oc*u=Ku9wug8h%$CY{AIwa;b?DZiPaX0dIM;_Z zd9wIFXWqsGWK@k-R-S1Wa=F~=#u9UwtqG~7ZR8q@V$7J}!UqjsHR@tetg56cMrKr2 z?-(C7p?$DX9~YH^gcqQUOgslB8s;I7w)Y>wgfub%I(j@syAdcO6N;gx$vG7$Ap;{$ z#(ziF9qnc!POAX9YMCxUO=x5`)5vTki{U~@BSPQ9yh{dM>h(e|@bQ~iL9@7GV=*P{ zg>_ftT9NY2%DBd3skjt@ae~eD%`G|$#uuF5+Ta(yxWIsdo)qJ-5}@cqK*a9`uIC@( rT1?*zeE*cIc=Z05Qio?(pRGP$Ti>LMWYb{}*sF=cd>_w9r)zQ*Lm263D;})0?6<`tJ}0&81mxc2ov9G`>q$Dg0MKa)3d$6 zJ;F{&21Ta1{Cenq?t0iJ&;jiuc6avzs{>hiCL{{-YDfb~Qd08be;1 zqBhOku*^+w$rg$pp@%OvH(y&@>suCg2x;pjCeQY4&%>=(yn)6iU8`xee6k*Pw9(gp z{=QOElaM?4aek>J(^J`R->E4vp=G8|&wf#p@LgO}lko9y;p5@w+{eSiR2SD_Dm4|R zN>4Ev8O~&+GLwmM@3;8fC*Jq*(jnHkcxd9^+aZyG5@ zQJxyJ)RC-@ft-e_7@rfABdGNe9@FS26s5MND1CYy)8h+0{;zA}4aX}3;uJxhMvZp# zuYN|6PtyoJV)N8rGfI4(#z_*LsJBH4=gq}ERWoxS(>VGJ~l1?@k$g@o}a zbtn~$x($v^LsnCf#>CjMuiqeHV|nT+Q1Ur43k8lJEs7s4NgPosAU#a>A>-g~leq1` z^*!`%04H_EzjO*2SM@d~rod%pQ@}4>FhTt$SzHj9Y+;n8=~9tspdO%PrMgTeLH&Rd z%S-IU<0l9-p9(PoiX`YG_dvs5(|25N7ZnDDnZD;8GIUDrjNnl;x%g=Dak)|@rBSGf YfI7KjL-}%>i_>V?- zu0KfjM&bG5pN-uITaSBTH2?BNZ@9CO>>v_en!mZqJ1Sot<*wcw1+PTD=j4BHgfHLkZ|y2~SbUhLzEpg+GG8oS93`HAKgv7*;qN)^ zmyK%4zcUU`lrF4AYoCAEd%|+F z%NN7u{od&4J3C=C+#2k@H{UbW*E_|=2)}BCwn?}!H!Jt;@$83XpYLBQLC8o zZ-6r01N$Ih+}6sdRR*W-w+3$k*jhfnZ}5wOt>yD`W!L=beuhl))4Gho0hquuKy^^v zwU2W+ON^-oPX)#`$0aav1LZP6k6~OoT5Ii_O~yN*(n?vIfbk3jjCaHV29DO))=J?_ zLEUe9NEX4;^7#eg)w28i3|={?wD!qU?}?akUq8EgaE-ZD=ZCGu?4Nsn*jhfn8FSNq zu(f=C8MQi*o%vPgmF<@?#|3gHpz7zUCnlwz`ayMa^|RKn*rRY}z`z@W17P=;v|K&- z^cmh@3D{abKh?q1w0wT5RXw}Uuig{Y{#4D2`LVCeZGS&U@bpA*E^TXCMjOU>47r@O9@c@;U&(Da*V{7^R0=zO{YiWM+bM-{Mzh-!;rh2qr zW}v&uXRwck-Cxcg6c=6>*jmb@9<2Kin>^RI3S`h;i5XVQcyPB0MMf*;+ooqF3=drp3>Xf%d8G zXJ=nro&Uac^`P>*`dKwF^n~h=z>!`P9o0jbZ7l=GF#aO_?DI?TE9Pfu`TSI?2tQkE zkxZKZ0jOuUdqQ;zKn5=WDSqi~e?Lo0kLITuXsxwxHqqz+TPt8Z19d;uD?gm)sG z{8aM@*jhe6!yeTd9@8Rs6Xa;veg;4LELw-7If|!x>JdkoT>YHFN%0F9&p@#1{(C@L zt{#-1_WD%gLaF8R%ishxtqz`>z@T$lcAsCB-R>{ts`sF;pH)*ms=3ysOCJ0Du(e{w zpLc%PT0TG3A){8dme0?17A4HB3ufrpei6Cto*2V%^`L<_2KD|j=soZJu(c-C!R4oQ zR4!Y~=Qn|;$JTmEuBT{dj<%n{PitdqX@0h!p>M>qdtwoe^2jR!dLlUT^!2l|2Uq9& z)BbxvT0TG5S>*G}*$egT_IY($cFj+IsZjP{#{2>@cfcl4og$#U7l=o#QaDo(Grle` z9)qdIfga5-2U=?bHW}}L0N7d?9G72?jsbcqEuWt=yT>oZZwhR!E?m_jDg6@0wP(7k zC;M61Gintxep)|g)GCAbBK_?1bDc%@d39QL*Z=RUXRrEMYppt{?m5|WvTHvjpx$2- zIC|6%TI2fpVQU#UnxFQ;7*u|DpUwA{t>yF6ekk_4r8SWj8e?Glxw22-xcrp8-e0aB zR4tltl`lf8>gOU8wbC42=ZCEo!BM^Hewv&1o~`Bci{R<8wHC=#omaM>>X6`PYcOY6AFxZTg`*O1_3AYi;B4$!)&Uh=H8QaDpk_nX2=1xw54 z7l_CFY%QOk!7B&0mgZOaW%$L6`}*0{gKNyq_4l@FrJBz`7fu59@wx}(X>0lXW{30r zU~BpOoY_5o8JX&5Q5UY($L>MZ$<@y}xpHcydTM?K-WVK!$+P4Ax%5D1v@UlZZK&9pLGve{sT0Xx3uMF5)nxD(B-V@#lnb4IOV#$TkLmC5C|fTrqTYx(?Q@u<74wXbYwbirxaH9y956Np2H09YzX;x>rsea?sr!lS z9zR$1`QHP}lSV7%Qb2WEVAZ~4plG^^`3#ci1RTv=MRp0<>h+5Dj2m;W};b@Db*5l#jD-#t~m>(`pf zl3pq5U0r9HSCn;1-(CJmF>>;LOu10yGS=_Pl(K)GSH!xtRFtCFL+qCr81Tb>U;gVn zRsH_d|HCR>Evj9+$TctKAFh-)?%J2a3#~@`mGEpUZH8xB=KjyZ6IYY!r1+C-C+=m{ zIR?&&t6J@g;R0?gjAEI6^3!AYsOlQ=CyOU8cZFdK1x1bTG{YdeaOd^VcTX>B3||W` z7xxf1xQDn=UOsF$N5j3|@NRg1ySMvb*iT0Nok9DzgPp<3E|;%{y;yhOdfV~*E}*V2 zz8xpM;mU(YTZ2LG@ydMP$Hf(@H*|;UjpAcm>JM9C<4ODMts6IoFNcjkw$HrP)rUd* z%$qldQP|o+OIIJ;#a*=FW$qI8M~^mST1LAXw&84)ej9VYa_q;)E=jgu?p*J@dE@4- c;p@lmA-2z3iHBE?Ur}qf_^BJW-nx0~ztZG;bpQYW literal 66532 zcmcF~`8!qN*S9%iipmgC2~nby>g;=+LX(OjQ$j?@R8b<8c@`N8m7&aJXfo}49cdIw zC`toL8Z>Fv^q%MYUeEPD&mZt!*ZE=ZYwvURT5F$s4WG~F7MGFJu(DXaZk4*1U!c3k zUY9_3H}%aH=H_2pFDR%p zQBuU!J@EgX^J5UW*TpYj<^PS=f1euvCnS<-f|C57mz)s%->wJ$FV}6SWwr0Oz&koP`7W z@GQHZ(QrtD+Q+dVv&Ium+Es90Q#Bcpwxtq`9Pf37Ah&0s6i;BOD6ho!EGpzo0{VOy ziYD!$W}I7ex%@&RdB2G&RhKfy-yFj=I|hiZKr0!1GM%S^gRDolCh@J51>?7yaKfK5 zwAc}VyEaI}i^*H@eXa?<-kApZE7##|>7Cf}sF6LoeLa!B7X`V~UC^xI7REdqXI-yr zK-n3J2_Zk}`Rk|1nPWa=MtwOW78^-_6l}o29AESh$bd$H6r8@`G(4~>#dST)xZWX8 z@v&V8L>If`-J7GV?}`|9@()e?eR?8y+_i-nQ!K;rs5$V>c?mQOq~S+Z38-C~%tQ-X z!XC>dAZjVu6exL*E|};AMGpC-Fds1~R0Ay6Xy`q~mYvt0! zcx|3w&<=8?Et%eX(vHV33&W6cXVX`)W+Kp}jDpY1$-ebtbl#>o2+aMRNG;>-E4YK1NQvW6P z`qsH(N6sLgsydEBCsNqALU+JK^c8XQ>VW4f!*T3UK6uF9r{`*#U?OuGPV`M;G8-hw zNY68Y_h6jaR5?cMj{=-GBmE-jri`2lv`IQ_chz8vnbU&dT3F6-wsejw?c-IqKZV z^ZZ0D8+Xx1R`c;hsu1nilTY8Be2Mi#T~OZhoVq;MMT4$vVtiSY7gO6t8+%WYiuhQX zQy#(4^yl>cC6-3GYO!Jl(}~Q&Bs_TIC^1}?MSl9uK%Mh)D7)eV6W^Nwo%)F&C1s5p zu1cb-$RrfK)k3BR&Y>~e&XDaYdC+vkhWLNEMKZ?9h`;|@Oe2fwhPBajV2&i(Y@x7P zdjgDY)Z^Jb+={qMhB}=#V#RwL@#_ytx88V0)yjo&>dtAP;@84j#HyeIIY;Nku3#r} z(@>^K0smS~g7a$oVDmpk-ks;l+~uK~s3x@zoLp;g##4P*cm5tHQoN8IF3zS~L&NE^ zQw&ToT#0sR5!7AlHeI;+9va8<7$rM3c4*T#`ar1~x4t`1y`{A1ISs9*(`d;|JC==7 zcOxKu+IrBMX9nMlKakG+Qh2|I10MoYVNY@i84RCFY@KF7&z(is=HWq2GPaP&glYIV zScMnuJr!GZ0V9`h139xV7_PYrC!B7BnqUQ9GMh}#H_XB6q{p}}*?*bu6*tt(pU-{vPl+IL-~ZMPYpmS`mq??EsT-ex(Bl~rooBDyWw(I z2sl-Ir#+7%tS{*&GUNcTbD)ks$H))s72?&3^4l0>zn z!cfj^56yEKBo}6SGkbNspfp|`UZoBZ%N1v#c=r`{TYm!+b(00F_+Yqk1hGFumZav~ zgkP@zNZrdku(L(16dEJOvVXBUivUc`!X$13oIXAWbIdfz{GNRBu%?D%tKW>$Rwl4$ zaFp)XQsX|Gg=7?6VmeJ2#Wp78{SVGfJuKQxUWiT10+SBtpZ{B(~~r zA$xO0HN>~xf$yd{RP#nB>~MUB2HmZ2F7FpPJ>LdR`+ia3WLa{gtpX|%T_CmL6uE!8 znSS5X!5A#a!Do6MRJLF%4ozsFy{>~~WlAXp=@+QCD<0Zzjx=o_>LN=XF2norNz|}E zpUz!%0h(z(mh5x@H|1e=bpAYO->Zqit$palb%BKXZYn)>fVgPr!1vcfD0Wg37j^Ha zhA+io*22kXI!-W+`VfBe{u)CIyoRwE`tFi%hDoT?0 zJ)SuCtt8?7TngB+2hJ}|fJq!43T+bPQTtJHJ=YxPcWl6CM^fU^Y#}7D~B?XBIVxVTjqnocMvB#E5WBZjTP(2?) zX4?CLr;{Oxd=-a%8)nisAN)YzsV`X<@*Zc-_J-iYQ}EQu({N4uEo~eRMbqE<;Q6x% z4K~Dp@zNRiYvE7g5O^I|2wh-Bc52|mehMyi+i|{Y3OyRW3nx3=V0EO-ncK=SP#&cM z+<)y@v+o5oukdGTl;VJsaGOq*UIe?(YS7f@m#Cqf9_O?rv55E1r$)5c60&T60@bJMvoG{}fJs7D-%RBBf`VAA{@a`zI zNgkr5?s~wxr2-SO@}O3DH#KaX$dxRcgB?yhTz{;Z%3UhQ-a>UaJNlXB2)5w0$BM)_ zY9IW)sERI?4=G+>hf1^6F#XVZEI*k}pE@RCwE>S=%#8&nUwzoEVMkVA6qL-J4AX1o zV4GYeiI<-Yn`h)u*DGR>^S}*X>?tNj+zIgg)?21SqMYt4aUqw(ToEOwqWVm(@ z+}~yft40f0H=$nA;xh|_es3UKrq9MCksK5)KTZ_b8v0tqfu;bN>1M-v%0}Q|0TWbIu*x0pX4x^R&X=+3t8S6iyL;0(BNqSaDhaT+@T<1|K}=By!??< z^*VtVg+=48u-7=s-A+26t%KcG#Ta#}kaj1RLt$?#M2rnET5c10@9!@lrkT57?t+wT16LDB2Zs34g~r%%j=UtZSWTy+Hs z55%C+rZ&=5QAkS{$KWq5J-j)40FCc;;_Z8LNpUcTiq#BKDw+=oBK7p{y~!Xa9giPV zj^hveVJea*gRT2dQ!fd5hAQ8On)#mi{98V%&;LV)<4?i&z^7nU5R0NILSVlB1>)2& zcEX)FwDplf?S)PFGA$1l{@xAtTRxJ2D1UUSz7J7Ru>j5oX!Es15IrRb8;8`vLeCt= z#@E11@r#^mI|}iBcQ{@Mc|}GFJE`63S~9!;6=rnYqv<+rbmff}*!nA(UX_`SyX?O( zBrY2Y=gsEkKPku5DFZOD@*9TEse;6YM$pMlg`nDrysg*#acloDJl*?&1lh(xZekMi z`T7~Mrtl&$&Q-?1qoE*sbqe>fNE{}u3dD`t0cd>49A|FVB;Sg@5sytWfd9hD-Ot%L zTS3`YHf&y^dF@gE@uLKs|lOmylanN*QDc4G73+RYkLy*eCs6*n^mj9k- zdDm0BGBwbCGDxo$h0(Adp;Ua}H*`%1gu$LljIVmg+M4vhqxTa@W8xHUVRi=Dofb=+ zbmHli8!9v;RgP}{qXxffPQv#~ne172aIFfr6|J^vxu{V6e2-HY&p|2Et%JX2?Eni&F9Y)Xn;gZ0+ zr_mzsI<0awg_A*F7({qYtkKNzHDx83k-<|;Tiw+a-_u0)kX z`nc=l71)<-iFdQ@Ve6O0cwpf(qQ0aIt4hYnqRmmb;6@hCUMr06?Ft}yaV=374Pru+ z)`L7x8K;P+!6*F^TskPkZ9E_V{$@K#@c{pd{c1+iWJRNy5rH=k5^&av$6(D-qGeMX zNWoTVbatM}HBynp$l01i+%6Kfu2trW23%*V;)dy7H8+}dREae3^YS|h5%M}R9P7koPQoc>-iWqDimad`iufz>|2LVoF5a5gE*;(X@>uAhm|hHc>-WhiV9Z zKM#KhC4qyhKEHo)C!1fGfO116^z6S&VybK~uPhzw7A?kY-z`BzE())H^n)tl=%(+N z9--<(9sFi7LUh;eMBe-mqR{?^T#K8;vwSbbTld2c2UR=KpVLVE`^D&B_F8PR*C)^Z z^$_P=As$&5h5y+}`F$!cM%vjEy8> zzl}ObD4CKUM!6(SBo8L8Py)AO+FVYcAl?gFhEsa?fYTfsoGiZ+9%^RcSmO{=tR~4l zxZnq)JNpBf=PAHdl~sW#pw(SyWvR7RXBKSDtQy7hF?Wn$uhm0s2X_* zV-GCBo^E4yUW+^oxytiayS#<3vMsFP7aQmreFl&E&qP=F83SN3#B2cl3(*B-w@4NNPm9Vgx6P0r6svfvCHTLhK>BA-_FFMpz;C~8~jXc zAFA>8e0D>-t$p}mJd0F^xj}>PO=e12CoavfE)oF8npUJ}C|AgoL2h zHiO>wkfVin#4!Am8*KViO}K5dfTy#Zj7~^}6RU2(GX;4RIV^&W4f~tcOwq!k__wU+ ztT^aiCW|9CeDLZcAtKV`1s16lO#x>^a4h;IncZ5+o(a55*K^fK!{~I#^A^WhPV-2q zn<%t9Ibx{pb3VrVmpj5#jxVCf-H|`u^gl7E29T_i3ea$gW@Rn}q@0`LF zE4lRG)^@OBd=4lnO$PF@X{^`_bKP% z=TQ|%_kV%eQX<4o)fT49T*a9W1z<^UDA{o$1A3EO@TRjYs@tr_BMW`dyUh(&*ja@_yiW4>t=F=E#4E81*v%8=UTS8e27eqai)oIif(=Qgc$H~ z!{rIjnRyA;IGi4iiMNFyWSJYP2aBL}LooCHdJ<$_cA`=t3t-CPOXwFHfpu~HaKCOo z>HK(%e!3Blx^`FT#lC9dw@r^e+dEDsy*faR<*cxFhYr^#=NtRjGqXua;Q_fYwxg-V zUX}Ym_7>T=))y}HWspK{ANX`VfMD51W?yG2uG3Y3UAdpJlrx>2ZAgLbZo$NYYlDWN z6S>z}0VbXL!aB@-NoNV?u!nakaSb;8z+j#dOkNvKRHwz^;hV=%DP0>ctLI~KeHp5phq-za5s$5K`+e-saQ4!a#Z^CU^m#9tAGiFPGI!wN>guadzWOfWi;NgmI z^xAC&v~jV=X;?;|`Zd64{VS^XW>ZtiyN_tppG(Hy7Q>$GHs;)_0a7_40_QLb7KuDy zJ(a51dpEpbuJvbnqWUK7IKk5W5|%`LOD5H0Yk>`rqK&Or!1zaQ_nG*JzyOUSBN5V^qw40ifVvc65>xoX5i_24ydn-)yA=vUy6 z1wr_4Ko^P*>haPuhKyEUL}+?M8L?hOE_Bp zlPS11N_PHjhxXf@loUyT@vYKI5+mi}qCqFY=j+kJ``&?wxC2>#xBl3zYWKL3PTp<{W#^wFAPVpvq zub9DoJDx%I=XAlwpInlE^As*9Or|zg*=YFZCb8Z9lNlBg;i)`4h>JB|Ldo`tC>%D2 zST93b`f3Z#8crsXb40L$tB1VkiInFlz}t3a1m8USfxH#TAU(*(>>Pwxae)Wq(EG(e zm1pwA-+aYS!F|l)9YLT~J`2L<@NjC!W%ei9!0GKzSyO8+<~z*64H{8!G_W2;e(NKX z>jgiPEb-N$WxTwuGLq$Uk0$QCgY7|=(Rzg-Y_l+fE$-RaeDn!9a+ODy&-89`$XN}i zxp#24>`bhlzKSalT!sO&=99|Y3|y3N10LLd;&W#$HfP(jr)5RJa7!NimH0riw^l+P zCxNV+lnEVy!{o=eRm9PMGANYF!Q+Yv#CS^>zRwS%D_u`Q`W+tQ{AxY^eBDJmZ=444 zw2PQ-(1sVKIXGL)5AU4zg$Oo)V4NB}v2Law(HzLq)uK$uVN{ad#^2Wd3P9h3B|+bjyQCr>C`h?i@tdw&F#20K=S1u!qd=oj`j&wy1gDQ$m7 z(YECg8>t?N{+}L_r#IZ7H%go4UU*9`{FH(6dl#T}elu~`$;PvDPQrngoz#g&&_(lt ziDzvuNo)E=4MOc`T(JaMU6_DVH}|8jx(F*ESHVm#QpBd6i}3KjM0)1oTw0U=hKJc=8e^Ce-0fzm>S8rU2eY7ICgm(t@`ml^CA33Xj~`iI$seF~>%qeRJO!-1SJC&Rsq$g_&2;QC$=%RcL3 z^RZ#7>oXU-ZyrOl;BHt~oB+2c*KlrYp8y5!b=3Ryp00l02zIyz-bf#&^9<_HPNxi# zW~Je@MiX4`wVo9!J=er*FeX=DUM9<>JIM5Jv$&jBqaO#`L9D`}a*Y4($F1Z&1zp_v;h;o8+w za49(oQpF-DQS*v5q+ysotj%@ZG(x{@3nv$rIib$p8IUmF4-9AQ!o#c<_@}PJX=(!K z^s9uOPYv+i*>EU&mQSM>KVaGoCjy;oft!wK$mW%8sGm6nY`Zro=CjIJ{GllpPo#3~~#`N#{@$;yyCzlP(_(}c3qIc(>*V0d>^0%rtpsLc5;+;^c4 zyN@=JCm(sNd*uMM~KLwXAJSG$3=7D>u0+vqbMfB-l ze{R|iOC%nW=Vj4oe|{MRnN+Zj(gO737XjFOE&$WG$zZH4z`eBj1k!D?{F;!14FkDk zgOCP@Sqwr$hBMkaex;|zji67oj*VUMlcY!w!5uLGJaQT{@}5D`^J0|MG>2Qg+C-rC zFZro4k(g+IB(7(EV(j8eM7GiudqOv3>Y)it`G;1f$Lj*!_WLdTQrrmHTLgH0L1MW5 za4Z_IGx1OJAlz1*%Y0bw2E!Kod{lh~t4fn_OP??OFe{c+tlNZ3hVGG26KzJZyMV;} zJPY;$n($rXJ&1;NQm>K;#70GsXP&A`jc#p&4TtYyCGo<%t16&aYmCOd%1|Ej28Q1z z!BgQ)^Fv0OkqQ9uk6sj%ZjLl z7-0avC%C|`Kf^b}aNXm>xI_jq^xJA!eC;k8^gLjq{%Vu&{c;#KWR2@YPSeC8OR!&N z4DP8c&Ff8~9i}G%8DF&bp3GeOyd7qxodz~349}nYNCf|iGri_dFtouRFPbYrqRYFc z{@Xzuk?{-^uL+>iV~Q{sdl)(-r@{RB)#OB@IMyFj9F8o4!Hle;mkRqQG)-0x+&|IeL zRx{`69VwWEJl+SS$fX?A^OPO!LO^!af(b2W-guyY+gFkyM8y0j8Ebw5;L;A zCK{ISl!OZ#ztG9v&uD7!Ovn>xWxSX5gZ5(rV@Yc8_|`IP*%^SZb!>2Vx+r&wZaA`H zsdzhRHbgGez=bjVyn1vBuk@EBJ8PpUtMW@4Aa$$~mE`2{Rm4rg8@)m{`rKtYpXShQ-u~3T&4En$ zZVILy0_Z!X0>}TQP$&0qF#foJT;G|)oWAOhZndXTl;=jI;+%;2g*0+q*8}(0_R(ol zjZ|~tOf*fLgDbSA;SRHYREk`L8V6pX?C4}1nsyQHtCir}^bkncI)Nv7GVPIYp>^ULHlh@_2ErE)Vay^I+rniw$6<ITaHzlbp*~# zdry?bGpH36re_+vmSxB^vIVWO+$#dpFxlLV>P}t;&sMF2n?^PecxVgRaaj%ac7BG5 zfyt01C_*QM$MExD78s5X0C!F@SnneE=c^E0F8IWGpHa`5!8N0Lr5y5fxg6+za$u3y zLLQkMLE|TX>0}Ez^b-|~~uRs8^ZL({2Bo(+my zWI$WS0$9}Q1rvlO!tgMQH#5J|h1=EX%(@87^lreL^)qm-YCdepI)T%Kor$|;IZS?j zjgdKclKx0dz_m|i;6IBrc43Mj&sy*|bKYHuYbz9m?NfN*6(Wv*d!k@tK{2R@K4+F| zZbSW{Cc5B`4Q?Cgrc!lj;9e4hRY!E;NSg#cu8P34k$AMQ$bpFdy^vfVNDoSWr6miY5VS3rX%8A+Fen z8%=+x58u2sQMxM_^NmEP>-W1CO z#6G8q-km;7rd}pkzye5$wWGSM7d)R#cz)YuA<^YKP07hQ{>Wy?_r)3XEb*~Tbw zvKiXD;~Oqn<&#DaJ>JW3QyQD!Mus@WbfuXEs3eBMH`5pJ>V*fY@#lHBr^iW>Q7i=c zj^W%brR2-s&c>|v3}}zsgLy;NV3=}%D$ThMZokrrtdclf{`QHScUXYRkM%HWFCPm$ zsK{gWzo7q@W@`8ECTtFqB~_+#aUxv?3%^Rjqx%Q2qND(n zsi4N!7wy^BOk>iVz*3?Bm7jkmeJ3k%dS4Li>CC_)jc}-P+lxLQ@;OgTYG~olLNv5V zg1Tkv;fdoo+dg|Mt2#l4 zCux(#psr;u#wx1A&e{!F9oPX4wM8^lWeFKGe@EV#Ttj7M6?$Dsfv>R{u;f}BDr86E zFH;}ro|*$9eMd2G;RjmoEQ2bEk|DJadWuuhVB{#SahgFlc~NZw0X`;RfoNZ^ir8TiE4VFCol-6Piv_L77We&@=x7yM2Kf z*stxQJ>HLStW}MsXr3a9vKHX~U7ct1E(F7Br$L}n5jhdw1kPW)X_k*3v%*k=$ZI5# z3PVrGXp3SK^rPu>!9e(KaRjDT9e~HK5@6^pfO%)Up(yANOj@MKRd0)Fy02kOe`?&M zD}PUgpj%?lec1;TYWm3MtQO9`th3Dg&9#Y6A2%p^sc=sPUd&?aU zZZBmXO?eBN0p9rbV>g{M;tN$?j=XhK+et*2EtsARgGZb0kh3bHaPZR`n)s5>8MJ3J zzx%>rW4a_%KQg3|m!069?GW9>k8$?R6;!SGG|b;K3la(=X=2Y&a8%}kWWausD)NS; zO)qSETHQfpQuFZF$pRWYs)=9kh=QSL4turbEIQf>!1QkMrnDF};#9s6hhnB;#?=Yj zg;|#{Db!v3C;-S;i1xE+RYF&vJm94!hyKxz%AnPb|KlQVnC-fj?fGh z2i*Vo3Y<**#RdiJ$6H1gRO#ts@JCgY-O-AxYTm=PucP$SNI7}ApMbKWS z!8I{E1WMluY1pYeurC$hKIthzw}3EW92p7CkvC|D%wBq;cq#EN@PV%OOzLpQ4S#?J z@6Ae0Hv3H)Oju@$D}C+)oX^C)*FA8JZWv(?XHv0E1(0!e5$3za;^&4Ilvj)g6Zz+G zG57=3i?;x?C`;lw)(J1N1)xBfLuMWo=ZXGR1+CO^Y&Kbg4_A#d&pr2p#&-@)@@fV~ z=LUJ&lFX>@P9uh@k)UC&4S9S$%o|5Vy7_S_&dyfA#L#18`MV7mdFlg=yst?Qbw3B2 zrK)6|b`G3-<4ulR){>^|SQMFh7Y>z{;ftyR^tf~x$_h8L{SzmWF&!HcY9+#baQ6f# zXKF)&fGd6s9L8BTES;*A3+DRk8cliANUT*meQQ-rv#xugz$#HV`a_7jRC*Fwv%7*i zZF41C&))%I=QgTn6#+I4B3x6?pY-FYSkmul0<(gn;mOJdFupqhyI(cI&%ZUe!lM)) zS=7U+)kdJbd?ESk^96mq4xoRlC^T*9C3B~l!n@Ip#G%sKupM@-iowXLbMS}=F&aCUV`uI`YP{$)u8yAxKH{IL z=8=Q2JtUdV9$iFJd$Vycvjn1VpJd-|yb4d1-HAl%9`;&594WMAFbN7EXiqWSWpk5R zf-!Xap&z96ZxgKbJ3{UEuYhwq0>FN!9tn8Q-+y9x3RgcYp#Jm#OsL*~Pr@8Hqsz5; z&)vgGzey$O@NOgzc8pM4bs28jsbmmV*bi(_Bqu3v5M@$6v#xK<>Aw{YaAVqXd^d|i zW|1S?)3pH)tkQbR(uh;fP1n0C{be{ze_jSq; z`R)|FH*)~5-oJ&9H5LIq+KB&R=3o(@13$7v3#vv#p_-im_KyXL<&-%PVY--f>wKrO zR(!qPg1NNBIR@?prD5Uq$MoOT%S@iN7oB$`nkF6dz^eyuG2fee$?`dl@ZR$ww3+R| zG1YUNNc(hfj;X}U%A+LROcfF)wUTn3i|B3N3O_E361B9WaMJZM^TtjVuD(LnU{w-% zo;U;Ozz`A1*TdEoC48LiJk+|0b1kxTh`d}r`*`Y6)VLMOlkHV1#qz#*-CF6G*r`s{KGfM%%VlyCs{$D?0%8X)op;(PqC0JY=n8g z{*mz>2AmF@!uqFfv|Uz%JDQn8cRc4{$g7o*J8uszY*Ay_z#@>Za)$J1;Jwmz!L?Fu zC{Zkn6*FGp#5uy)XSofJt%!r;V&-uFUOo;kOh;rUVv(Q*W0`drx8#X)TjHnVh;b2& z3{N5(cgxb@)+LZyc?z5_L{Q5PVV;EUR_qm>k5_k%uoD9l*qG1TfbBBGvkQuF`QZYR zu6Phf?J{xN1q1fJOa^v-y$cJMUf|CJ7BDw#GP7y#G`#yQ^o3cwo|}+6D2i9j zpRnq+ml%N?pXiskObFK9fxfx95WIaS&W!QLflU&;f3|l?!x&+QHVSbcsLO$=Tm*(F zu&^lc2WssKAeU!^GCu4CM#D)TY<*skk~%F=*ec8R2FF7~wk-sXSb}wo7;JD9h6l|Z zO|rFzf$S24?bcH;ty>qDJavGt`k|OQ$j7Fcc=#CdopOa16Sw;fD82PKnvqv*QSwrV z9nB}gvgx?XwT8Gmp9ACdsqoIi2Y7$xfUw$GP(~C9*@HIwLF41d$ zRKY>1mTDbrr*c!LFvYd9Q1CVi4IAv)9uYBG_iP#XMtr8HUd`pc78xg1Uk2c8IG>ZA zo=Fo$zY>YpFHrL~54S!lp;zW^!$nV4LRm}s~aS$!Fjw<{4j&!fb4!gSDE>x)A$%6z``9SSrX>6*S0lsxu@Y-i(1)qo?8Ofp3M z*%9>RoR{S2yqUD{@F}wJPYrZFsDtRI&LnPa9o1Q{4KE{)LjYfIt9)obNLJROzd|RK z=uIUbQ^CXL(uB8`8BVlv84a#$N;%|c|C?GS`WZK?OeYcz8pX(QJ;YvMt zVxo>iZ<|3eLkq)ezrfM%X3}=x1N~H{!wMW5X1=Qka%-13U{=s^3|c-NH-HI(e`cu;rHjo6IHWUjJmPq2J z$~m}Z_eWUs;UdhAaz%r`^&oyKhi%_jj;rbpkiXUGbkd7c;P5gS`qsLFgr^JIi0uKf z-vk1z^T6L~fUonlrT2vdu{_67~F2&yGqqO+3EP|epV^rLn=A>z?w!|7Uj=7<~GJg}yV<~Nb@kLUSw zaRgkm$|k1g_A?s4MDg9_PG;_ELKX{$;rs*L7=Nam4hlCw!fYPYYWyNr%hZ4|OD1!5 z6Y$i46?Rz{!vf>wc(1IJM*b+m#wa7SH7jL#<9w~ja3y`G-c26s=i%ND>p=ajJMuCr z>BabFNL)M@K6;4LC*~1E$cuyer$6Jxsb?ToZH!a;GY5~S#3Cobk|b{JU}e?IAYNh~ zy!}?kPC01Sq&OZ4rc*_D*WMez_1wE)>GTVaH7-FTP8jXFCIM3>jH7m{Bz+zyLB(6Y zl5e*NdQNSkG5I6_vnE}dOjbPO#g{w4t=sT@H#CY~aVwm_2^^2F#IcoA`W{f=U z@*M&<8AVdrcMvR$#31TRH=7@wMK1)(^YpYMP~y-*T68=XP6-@_g4?TMi?u!6)((e! z84JFSCX@_VIT2gGU!3SKX(V&*MX36^9~-S~QJvdMoH@3z@%nDMxYh&R+fZ zn+iHBR>74&hxxO#29z@zXd<@-mnm4{g8Mr3d%Q87{c;4p)t#qz1x?U)uphGbWRd(! z2B_K`4pB0Vxa5o^3>>M1#BKIi{%tO?FAamKH*e8bIeyS|csfrcT_3V9-o}3}?_lBr z4`_bV%;y%usd)D$sv9Z5RU1`?ySZ2CCBZhhRBr=2CFDUWnvVxnredL#Da~9Wf{W`j z>9g57w6fR_?vj7xV9`9x>SQ59bPXs(lws+EL45MN0K@KFhn_4haDs(#H1-v)R}o#h zT<|2;{44?cBXWHG_is9LQ8%0AxgLyEDwqqhW^j7#XFRZmuZ@efL3M`G?P&+`%_$|G zK}9bLc-oVp$Rzwr_rnXNd{SEH4CNM5AX52{b=LX>F>`N{yl8bY>gR=9d(`OO@#SoN zi3YUQ-6UR~H!xa00~;;E!B8%mHP%j~L^c6x-R)q}!AW4N^bB=uO3=>n1*5ZMIs_cz zf`#f7bk*lG3Llr#DGk@i@_&_Vo#PGMby11b8XPB9<9r?I@j_fFtWCDXw~^v6oiI7z z5&dfKi;2;T;iHK)iI}R$gm`B_={jrd40nXqwqK-s{~mJV{kg{VY$O_bFX4@9WuoWs z6f!&EDwOHTbDbUp;fq7z@N36JuE9x`Ra$!%eqKAw^bYn>r|Epm;Ee;;7Hva`wP85v zloco+)P$G9%P>4Xnof)2&(Xyb@ssl={+bePD%?B?*4OT)2UeXxwH`@csr6Od__!B; z@^uKOriwsq>lf-7*J938}Wvo7SC5p99X?oxK4H%9LIXR+r1FRz5HqWYyO^!X%?7w$OCM4 zf25;J{$b)|6=*G%08Z3>$S5@6W117WAumj_&0T(M^YCM>!bA%g1TI#W8Z413WBNb+Dcfs1PT{u0?jcgOyg|E*I!okZ( z6l>O_xOpL%AE^ePC(+O@c@9LYGC4HY3}+2^g7u9)qWW44HyHES9ejP=wRuRbjK0w( z%RgZ74(aH%5ESq>gn%=9QFHM_aNy{{opx22A9W1soQuKBX9;cL?>B-XzMif=3GU?! zavhd*;`nF*d6_1LPWpN9e;7LNxE#MQj0;6OrLB_D5Gh6Dea`(x3ze*Fq7r3Aq)6J7 ziiUPd3n`V7)N}5)($YW)g^EbY=7$h|&;NZsy`N{DbFS-qUH7|%C+R^AXH0l&P3W^qApQ4=H7yT63rE>i$c>{Nvc1BzO`y znSXcsTXw0;b@sm#E->%LO{#J>4CP8y(Qm~#(!XvS^n5CS%|;Uhzoid?W>z~kO%21% zo5sPll1nty&Il^v#;_Tq7|liv{-N0q@m-Dp-nAQG+hk|>Ptg`vPu?fU3%17ZW4{2` zZO3~n9)n77BJSUtLhLSw!6M;l0@Jq>NwPr`lYP8^_&j?`^ON4tEq!t@w@MXvNPd9h z2_fX(=NTx!I+sZ5tfi_mklYb!LC=wJn)jm|E}i4_RmlUSygLMI>LvI(;@;SLFr0j^ zEhqD|)rnTxY_L#V&A4YiCoK_PWHjzMeaiVxN=;jF!|aWa3Mue-ECxKgr@|BQ&G_QW zQnJB9f>F@?h?m}_lH+<+P&L>?-NpWx=bgO?SJ#{eJ@;tXVC+Lad|yKhjH{Scrdjmu zrf~Rh{yXgVnFCbfGd<$H4Hr3XhV#p=F{O8q+M8d;!NM36IT?(4v)0k}y(h7#AQ2wt zZ^cB{`6#pH9TOHV4AQoW%qoW}beWI>#k0lvE>6e5W=cFJoiGC(GZvf&iC~w$KlX6Z zrA$L>w9(GMO_j$X{Z%J3@@8)J`;0g6$$cG}BKiUb-LjdCOcmU9@&F!P9F5o#Y% zJ^4&K-=&d+ty^eN?{|o7e1o2E9N}@1B7d9OD7)>jBYx8igC}+G=+;pY8kTGgvr|1; zv9@FMNlQK1+&-S}&JSdz4-F8Ld23)xq9+`?p~(+YID(Jg1miF3KW6L|F8kw_%Uy?8 ztk((`^0{ImjARGE+6RZxR8yKiW7A~*+@m2-G*1DZ{YwU^zs9gG$Pd(-5XVbP@Qo(@ zAvQl0`4;b|pyPP~jGx>L2}k!)N$qH|Abuy97+oUg!d2k2>TBMSDqASvcqXwK2cb&t zJ>jbzLf-AeI5NACxQ$O`J0|6mnahvDaCRW`WXUufDTs!a%@XkEbOf9lTu)6JZ__Il zHwoQP$n1&IV{Z8pTZ$)o2*en>i{~J z$iWm*51b?Cj>V2gVM){lRBsN2r1{~H@iY)3tKzVFizAG$4Ttt=%V60`b!3M8VJwBk zXeCi9^IHp0v~wR$sD-Z+nswEUs!Oe$L4e1csU^5I|6DcH?*C7d`xV5Mk2#2BuGipx)E zTd6zbPdbe28=8so`PXO&zQj>Flq?()qKz(vs2x0rMFX~Qe&1De_NqI-D+4!7h1^YH@tI#vZD-VE4zkI}nQ^(6o55WW*m z9A;p zINVv&Lmz8#`84HERN7)9q`l3eeNLIw=xZ*@l2g#VZUVmbdR5Ig2!i71cW4$+N`ImY zRHo$OlGqKHT%SQ@b?qt5tfU9NwzApbXK-%CHq7U|AhDcL`fF_}w6`YFtvdBY;cyXH z^Y60(1;k9qUkV>?cH#AgY{nzUhk2X69R%ZpK4>5}M*Lo`1o@!@IQ2#@$n2ZSHhtHC zj!re8PSYT!Ya_?aQqp#zjeMHnj9ZN_!pj-r7~F2o-2Y>W**_XVA|M0!>N#W&4TjS* zCi8i<*2Mct7VI&3NwN%YH3c=6QI3xf)(^6@^;uVmhwnHq}j4 zWP4oa;g|7Wso1O@)LZ3`k1h8@(4|l;u8tw2->%RL4~}A)dttTHbTM3Yvx`|PT}=dA zG^pRN81}4qD!sJS1z%j#5|pJZ#4;6*3vfIH$)_dp|&L z4BTGt#yj~**ySRL`HQ~5(~)i{%MYb8lhbJ5Sd+bx@L zN$nV(sf%P?IwDBf$90`gwbXmqUWcd+IqS8M<>PVAG07-m>6a(*IK&AGS}% z0&R7u%TUGJE2H6JRyV`+rhB^DXiDA%kJgcG!5A(zEUwa61(k}oG2Z-|D$NABFws)BZ z4`F`quOJ9)i-3Uy16&ZI0Q0Mq`2nLqyH82t)eA<%^!69lQdkj|j>Rz@i2|tI;Q}t6 zJur89GqE~-2+qoz;MYg{VQl7ohQxp#jejLkVu++-*4 z5ANY|zMg#8X)T1xc!?gBD50+%($L0WH7(^B&XdBSFgGg+vgh|w$D9FJ?p_CrYt}JL z!5NZJCj&D5v!L_55?t}Q!PuQygl?Y3yzP&oh`U`IF3|l23va~{BU@caue(KM_nc%u zi};|CKn#of{qXv&gK+t*xWFf66&X}p4e!o5gOcT5RO#D{V%1lPE58dz+w4u+S%X5LEj%}2;FP75|hi#uCMUzit`edc_>p+1Ol{RP|n zg^8-vLK=~BlimD%78a!~#Z#ZFh|%HqMEpu9Rw|8=7l*y+7rF_I?M%@-d=b3%ui%}w zs=^ZAyOfvx9L4=QNNTGZ6%nsQ3(?)kU;7{4xIaibUCx8@em*QazZJH8-U)@Sk<8Rm zbJCVwfQdKD;i~aT*ySoq4d;zV;WzWZTYn9jnZ$rmy&@UVi9*ZFYfv2ah!`i!!=a{j z$h=ufD+p2as0P2kc&|CaA2CAKd&AXP9lCJw?Fh>mk=77N$gToP^_?(+!>U@lWY3vW_>4S=vp({o%jSvjw+m@)eb+qcj31=dGPOiKD>N) z5A*8Nm}7cB$%yGovfjW4Zq+RZ?bkbLDY*ze$Yq$uJs}SI2&&dGSak6>ac`bRXUyY6 z!JpUcp=}Rg?lC_iS!oaFd`-w4<%xJ_w?Dgq>+4xrE<|TV948KECNahDpF%`*D>|<2 zB>#H5SY1^oo|X9@`1e5)CWI-`JOB2wIudWF@w$5K&H4(Xf4Q^l&*@yZP&&@hUx2Hx zc+o|(%RuK_6tu3ZB*l42`G zZOQiDAGl*_4Stv0h_5D@fU9!^4s6S2YRfqG;ITD!s@lNUs)@KfawqzYFc5+4Ve2#z(0ldD6wnfzcI+U1r6M(^*D(Ib2C(0}FC*q#P$ zQYNr_nF6!o`claG6${H1^D&{T2dnQrB9>oSIKIyc4ony!nH5s_HDVC&+G_*n1yPr7 zU8;Lg8I5O(3dRnWQIiAtT!tjPsui{n*{=`Dd-vTmA^sqy{y7Qr-lUW1Bk6E#qCA;? z<1F=gp#s-_Ob`@pl|h>sfnb!?O!j~K$xPfoPQc_fvb!7tY3-Ii_5_&&J-dO-ZwrE( z^-b6-97(JHTEpSZ3$W7WBV%oJmq5aAa68sOL|xNS%_ap~E$tWu&P!<>^QUS_nJ_nG zIvMj{gGWN2g2>hZrqGw`k?H$FGp?;>Oy2a7j%%g#_=+ehYHkLj2cOXHVgsChv4#Fz zYXnAK@sNK_iB<(j3bfqk;yhawwn_arIU#!k0w?ujg8w~C=?G>*{)KaUiSuw<-AK^B z)tYbVzn)&6ILdtLI|m!@H^WJZHS~>bCw;j46gc@-!O1ZxCg0~WyqF%vR%ck?g3QCX zp@TC>9UoJPaaEufEChFICzAZ(`Jm%9hY#WDG(tNYcEk+edYy3W5HF>Fj3(o2-!7V# z@SAz(y&G#||4={c1YEA8L_W%8!Mwai-j(NT;OqSDxOew$T5n%Ter~Ej?UOI*k`q}N z`Qid~UNiyTeZ9fnFL%NS?maEE5G5T)y=a)_KZcXT@$$wC?12_V@cVrS*Trd*e-D0B z(d5~<-QyK+SMfsFVJyeL)Spiln(YU%1zD8nYx3_o-2?xqbb7@{kA!u|!^jC8ywW)t z^iwXAPJ?OWpvHg1y3>RF(kw^mOVMy+tO2Vo+rjhZF?N;fZP-xCCwDKsU^nia2YB z?<%40z!-gb3~U4(hxu0o9qV?YUZ6DiwXMaUiLq$5j2x%<2gJr_h^mVOL`R7|ESEunDA^9uE9K1S>Zb6{7*M%=bi3kA9QtXkV#$Pf>P zsoU=oeZy9G9~X>)ox7;J`WCo9K>(7QFT)L&-SD_#h^dw zco|mH@DiJ!7R-K`C`I}$9PpMw3U+Y(V+9R_Gr9`6L=97+`2;WB87_+Rk;j&r^dvKc(0n;I7!0{l*{Ouw>Epzed-vh)iY@tA(6#}t4 zb76iSz_Xic(D1SY${H&0g*<2RPabzh&*&vkl&en42T#M4sx~%U-x4!}pMmV$naHlT z#M_@c={W@zSX!+}zjC>L>&=%y=Ttm=UD8b+7-piJ-D42`vKp^F7o+Fj=A+4rG<@n- zN=SqT8Id^)?$T0N7N;jDnDhxg#Y}~6{UThE7fIWFoQZ0bIG#{=M4bZxUDl`5#?-|q zSU(#kHwZwkVj*i9ejbxjBXK>KtzAC%Iau7<17Emr+13voWKvE#m5&-G*ti3RDO$KjEuYx_AuZd!rDoXu+&da^24)<4Y<}{&d(s<@H#+b3B!-@}MvZh$@ z>pKxD38Qm{^!SIK{|CAbBUoK#4KX!Z(2=Z4y6%2JUfvmW{Is3OVlH0F`i$2WDL`t0 z3c2x!;NLsm+&3qh2CD4DWgM>`B6Jp~UCRQ^Y9lN^IR-uHgM_Jn3JL0K;q!VS!M25$ zXtilEbdD{e_UXq^tw~(4eh;SuFZ)P-&W%71j+vFYhERU@E{6E6L;okA$)v+9Id$_E zxsds=+M%u*!fY;+~t4`Da*mdFPL1P5dx(O;vkZ3N+my>fv%)mcwpWSa&UAf zXkHzruV%aiGskFjJXcBTw-wR=qk4KzPLevGS%C#+H$d*_e6~DELNJi#4}ROGlG;y! zIF@=J>k19=(%etx!+Z6}ky-Xo?6a0yZyqH9HZi!RVmB^3DnXwrs$t)pnfUqnY@*XF zCMY+(KrOlMq+D)q?LAKqlNvhdx8*=Y7pLLsO+)O_71q@8-ZuPoJ_lNSI>;VwZ#Jzv z99Q<8!GnX7`QCp9aBs?WdWidb-yBmYk2s9iQ&Z4&{X-~D_`>Cl_TjiWv+3FMjpRd( z8I9{3B%6IFus0kyesASUayNSe_DvQ8=P7j{y|fJ?95*lrlRc~a;)R$+b+zWYuUAto zr}xx^JKshMV(^!nEG&@WVdw7;?5W?8=wrAZcr_Zl+cGandA~7fnJEjR<8z?$eJwdB zDlFK?9(vwh0!xqiW z^Vu+&t*~O>GQpyycd4d^6KSxzNJc!=L0Geu=CbnqXEQyK(M_+`)(mEfLxySCktY~v zol%Jlx0NVA=mL^y*~ukpAF*uA{R;a$68MnMNOqC8Ja)WUg6{5xdS@4c}eUnCMx24rls!DL&F@Z4u zgsA)B%V-wpkNYyE!NxZO4w%W2!b9s}TBIm>K22WW+na$|%TmGD49VrE%lO+qL=m%n z2=AQ-G3IU}lvpjHvDzl|<;)xKc3Lhb9|BtN=L7k}Y-V4qQU$kPt*93?hi&z(#clql z;J~IBoH$Yl`UR0Rs(KNc9!`XN;!|;!gDWg4xk~3uZY5hEEruQ$VMs2Cz#E0JP;l1; zw2BRpgsNc2lgqGswJn~Qn-5A(A>bDD1JvV7==^;Sk8i=?-&1_e~um8##|2-{%L4XRqPv ztE)IpMGSZIuHs~mo%|m`O8hTBKfu0)!=RcMjE@pj1d}xlNvE3|jOX&He~$K(Iqni1 z*Ts4G8llj*e<$r&e2bP_mclk&B=L7C2I>e2%1qBf|Dl<1>$D{frv za{^|KpMh^5+#*+#O6i@OBXmKU3n+7X`-?~Su)8;v=fC?bQ`P9qXt0|Y(~@e$_PwOL z{46Q>N3d5d1K-x4KzWNG_LxBqv7Ks;DZlQLf}${Td#VN2I9t(!rIpm^*b!WKmt$s_ zgJj$3y_~N&7Y;J(F%g!L_ng0^VG%)6mo6gDN=`!j_sN{sr3RaZ^2vWH@2Njrh251b zXobF~5jQ>2$i|kW#yXNilRnZTQ7>_yLMCA?X5+@1Kr&q~iIsFHp;H3Qh%9&QJ-)q! z1)CPb3iYq-Th*g@xrNJ47S~gq?KkPu<| z{`>ccntW8@J{(*hTFyJ#)jtDvh*YrDnbS(^|HHwOSdJOFMBge2P`$4Jy)S>G(UBbU z`i1K*JL*AC-unY(7c}{w-IfFP-Xgb@HR<;FY2d%*0THw+3i|tjZ?^%jVglIuZH2AgBE|vYZe-d6Y)dk0NNg3#hBVFf<)pDvM)c9 z6c4mu(cu#q#H5hXlstTCn+UUAqqz=jYuu@?Mw)&)<89i{EUmC&RF2#wr#anw_5E;E z=#l0#v(JIwng-_Eup_RR|B`Mj5#iUi$KsG&3LKr_&asAv$)J@wth{@LF_3q|xb zd;pCPD=2$R16b|tSTS-ImMD6o=~DsN`j{faX}i@+T$ne-iR9c4b(mrN4|3$?1uK2z zXz>aefq3)=(B3hj`pmOts9pYy{;J%~qeB;P^V~MP{xSj2tegs_qLt{~C=22no`96B z6FtoJYt;k{Q(vh_NRC{OmqSaak=sw4WtWIQA~Xaw8s(U(rGdi9GX!adHEj9MAR4q& zmPff>b>RsL{Ekb3U|=>#Zv-Adp@$owI5~=CHyFXQk9FW>Y=fdLfl%|Ol%(!>icMJ_ zoaeWa-7Fr0f-fA4{K0@2oKS+=u?=|U@EmOEdPle0W#Y>7?eyH*O{mc*z_%xDSe@DV zaM-Pj<$qGg)6+|cw`>tgAF69kN=lpKv;)0hZF>qYd${cQTkcs$&lm4$~z|APj0GQV$BKwa!b_z%xTk<9sN>@Ig< z{y?1=%$xOtuCn_NvcHUBhWJBldV2&dB<`W$l>*LE3Wo7UMvxK}jPdK*aY*MWiF)sf zgRhE!-^j5hx?9NDt~q$Ek1~H2WG`o+&FGPv^ ziE$*5En{j;TOq&e9Vz`NL+`1T(SNN0aN_A4L}OY0gpW7DUaE|A|67Q49Q*ptNgZS6 z3Bhv}3%H|K&MK6K(ERB^aCKE2%xO2ry+exFnsgaLKDJ{Y=e@`o%tRfoZ^E_YFSF>X z14b__rcKUGyr##cRN|>5I9ukz<%BjYlIx?{=cfsNh{}W2<)!GVmCc<)r0DrsTt{Sh zFRho~34fAx`Sw$d*#69e5bb*oer-Uisd*j$##z!bTX%?#m;vbnkEqqNXJB_H{1%af&)*WkG%PM2JqVDClbpiLr|nR0gSKT$B9F7IM?$xT^za+wBPE% z!*_{zZmgD0;8taO(;l|<_Da0n_kw+07mXuT6Zs-`gaoLnU_ctzg)1Y5zioPjn_Lt z7BJ-Lcfe~~56}U2))z_sY}bXr4E96z620p1pTR_# zuK`~M8^F-hAI#@|g5#4Dz|OuOo_qYE!CrTu?%N=74!OnhU(ZARM-Lda!V6$hERM!C zg>d49D9$=x!K#;9f!c97f#sQt%);I5gcE09a}& zYCrIk_%`&?fT2t_TXHL=w})Z`_ugw;N`TjS5q_oed}_I$JCEBvrxmYkQ2IYLLGX$y za&2l2+l?_rA4_0Np7-Dv&Q}YIVlduCi?TIM7`aD=`F%7U zyRLHi@#8Paz0MSz_Q3{TdF_OA%||hJ(@ZMTFD&@kA5AMG>PYCb9Gv%J8AQEs#-(}N zsMUcYj2B$t^^F~ZP~Cl)&O}rGJC6I9d=X+E6~J2lakP{v#e)k(z}d!%>&D6jlRiyY z9vMkB%DK)Qz8F?w6 z8_2}aNhGeR0&d<6#na!`Lvd+25sc_j?UE|0{Dyl@Dnp&j?i1!HjDF*G{X zjBBD#xhRwc%)qn3&Yqc2SD5XP=mtLV7MJLBG!&aFI3t&cfeoor(xue*@BmgHStQy zI}&r<4Xh&`!5X_a@Xf}aY(5f$uh%A%oS0OU3|oaKKk35#c_x_H|C{nhc7d;FGZuw* z(u5Zo&{d#-MjsL1zL%l{Zui-YvjkTgZzK)6H&MGoTHro?5#wbQL@rnTrC&wVnBuw7 z*e5*1xO_2#Pi8Brga)T`T%S+e0uJHB5K&B+^%3_d2Ep*}M)p=$HqF-c!Hx)fdWq}x z%l&;1HcfcLtLS04LXxdDywadfA09E*sc#W}KR7BBvD{2_$k$-cpk(m*kU;J?* z(S@AY9?#Tm%fV~06=d4;3OGH#8K->ufZuhrv2oWXoF_2_heLiry3bj~x)Eg!JlBA3biP(o@?+oGU375tN;h=|1&xYAAEOl`3n zj4qd_H%!mbzLh-O=NE_XMB-s(?*d{G{0Lg>=Mtrh!RX=hl*VRl0mtfG@@nmWa5H8G z&9ddO{7VTqeo-NosLcjCkO79N3(&YO5w{I7aHC2GmP%NYH466N*g6Cr2#I_6L-)>W)TW|{_)|6tsaTLwq4}e|!AJA1y zhHIKYWJJC3`H^wZ9WM^5jo%@OW5i8X57BoFOOC|HV@_fP8OlflEB|FiI-&4(E7S}VC0W69_QXd+Ux+3x*<2;j0n~o74r832B7aj3G#M6f{_z?n&fx~Df1;N0$eT?& zKg{MAG@WEVX>1g%OkD}8Xpar6cH#QMZ;Y7QDAcHBR38y5VK$7JW2SQj#x5O)`zbf) z10u;*hj5Hk~?D-^nzlC@zt_vir}%?qZ9V-8Ti zzYIj@Z=*A11i-gBMQ3*2B|+s1(0KnLgg>mLdn^24hk*r17k#I1*$kMQbPINOMDjlW zb-*nv!b$G8$4p4_4s6*yK(_GQs8yZ>DC!%a>nINjPW^*kzni#g>JN1I{GPG?>I@pv z`VgmELf-$lK@P3d2XE;q&@N%4dUed<+%;Zvu$z z;n>LI>$!PAi=nYIn))p|0djXg&{N(l?R``Qve&c0-QYfNPEH{kw>+Pm4(GBas_Kp!^e?*;$nW~w-%C^(xT2k8w-jJ{_X`-gc;j%rU57}*A+%k-7_w09;HDHP*Z z?|t+~R41j6Tfift1jFM(skWe*Z0guZ#R@y1f6*y=&{7XSDb{m&_;0iw1M%01VG^8} zfUkSG*&j;!sCfMq84w@FG|w9P`>7>rfD*es;LaxIT2PM#~f!Kl%&;Uu$CYhEHr)!I+U))dv!&heaPyZacZ$S_`@G-Gg2#h`W~b<7q(^-tNf;-Mo+VX8c-OR@y*oCZq80VJv#y zMA@t~^kYrA>vy(#?eSvncb&u4TxZzC;vpO_ScS)@coFr0W#r`ZE_~3DPxrV5!VRH0 zFz2!nWVFsV&m1%ds{Dn-rnb`K!NHIO|dd#t=~$f}+Ed$eTKK^6ttH zI$mNsao^+(YIBOQV0Jt8dk}+5$vKdb%O&qzR=~@R7tli7p2~^tAf=mH@YThATz640 z{tH-8t>Q5Wc2^@Lb#YnPSuJ$%=j>{$mQ;*3R---(8&NAk6Z&%#fFvzLm)KIcu&IVj zcm7Kw&*;POf%B}+>lVV>I{`H$vWU&}B3v>v5~fF#5!31|q&`fB{&!Xb!vgO?Nq;7s zTP%*N=bk6=IEp#VC4?#KaNkpRT$)gYqjTocA+Ceh(<_uHs+>k6 z|M%wCWz8@n#FeH@_`r6C?uV0kPB6bU4gUFDVSL&7U?RQ+uC4t=W#u%mdHploWhF)p zg>>1xC=cxF5wlkL}Iuvlu0Yrhb6)NgwJV6aj_RUt-y`Ge3lBYZ-1lT=QP3F zrCUMS$&*fB!9wtxHFUv|WT@CCLcXbvL+zW~tQayMrhOX+r8kDipW~KbcCDPX%r>N1 zg(5s1Yfp@Ql*BxJPy&)=exPSxk6o#SRJGgzr#u@%tJ`PM=!q<9aV-45M`39HaTlzb zYD^v8J27TPI;>uZ5d0w_WeU)0oeF8OI zBWB^^`}A*GKkwPH0+9JGAyC!YjJLfSNmBm9z;l@LFH1Z&(IV7X& zS#FNbw~fG>eP_DDrBLUQCW-%-j+^cMY50G}AUtUvy`;aXioe%{IJt2AbrzxqW>QneJ4N%_4Vb z^2leZXL=WfVi$l|PYW%Zevx@Rp#@(YRw6NFQ8ZMvfDO?U{O<;EG=h2N0B}BAKi)0Q9 zV^aDaa8QyaUuP)sN4~BX)GfV+m5U6q_qaL?3(DaGKM(@ZZW9t63+y`Xl> z7L2#1uwH4gke_Fc$Lpl<-_Im`pM9DB?MZ=BqjT{0q7R0@dQDa9MR8B@dAc)v8UB(_ zLC?*`kf~Qerkr_BPOeKNAy==^neyZLX049Z(>Q!@Sm6fgeXh^OWhBA(2XC+=j?0zZ zaH4NYSv=)qhLMv0Ve_MtV4m}cnQh4B4!38+)c_x&)mq7>4cEi3@68yzP=uevuf#`Z zC7Hb6Tpm4j25hdLkF8Ue(x%xqWGrzO&JCN5n@Z|Q_DoeMoRWzN?0&eNb&1SW>%?;3 z9bl$rh_1ew;PmY*aep8IHFvlUdDk@N-PkV@c{l=%#mtdWlHgze=Z71|ig>5ia5_Wx z6B_P%mRKom1m>a^nw++Q^VhC2bGn83BZ40&mg|9H4{nex??eUTF3upL6IVm4END6`G@}zm$equOJYCWaAG`KP0z)5$}TvR`yHi38kzd>5%BIg z;riU>V{F7J)VcSO)IN{LJ-6;c`l7#>zib^C9Mi$^vrN!Lu^+l@Gf|s-Cr1s6iNo5n z5Hj-@*|{r^xnb9cYpYhXiW@$IT!;^><_3;UJ#`-JPyC?89&&iHNDN~;G(p*{pDy1K zMJD4TPHmZfAQb=He$Mjf{|(`Pd7h; zR5f$C5+8e<`fr$sC8psnuM6BcI1SvKjuVB5FwokbOx-=IFeIswF(=w&Cb!RD-0~5N ze(u8sofG+a3;YR3!;%h1O|T35P9tO<(kmJdsfA8H_T|>lJNg+IFrjHM75;EDp0d0@0rJL3i9{@33tB?zUUySJ=Viodh5^2;n^=`7a|%70(okf zP?gM`7!b0J4qj0Jd8}l9p5?mExJ=QDC=+z{?ML^OlLa>)hzL5JFM`VbEhw>K3{9t< zhSW*ZiTwqxk8E}hPXBE|$O~ucE31oxHy)F`-5aVsq|4}UxdWJdGKblnD9u~*Xc6Ke zBlxlQ7JD&23w?yrNYMj%^5Sv}Iq9Gz_-YZzCRy1**@q)IJ;n|c+IGODDc^~Sod&;8 zNDW>tD#5?2Jc!o$Af_+sK7?zlL5^k+ge1wJo%&38nSTZ!jz;6P+6zQma0fs19z~}s zbMU~wy>xH+aVq)eFlIJI)1f7GMC7d$=*KkEc-u2Hr%;KNoHB-=auTr9b0<~zN(HZ; z{cvBz6puP_9?IH1Xrq`+|0LCr4+aTnFg&05+`mg?%sW9rAkX#lbN#wg%jvq)<|x>B z8D*$9^ep~FbSx@?IhzVcxXiTly9M~xg!9JMg+h<OP2Q1FXBv0PPvN5wh6VEmlE8iA@ zN>2{eOZc+XHemGidns(l|CoG$d|@=Y9;iT?pyHz zL$@?jD?2Y%K8?$Yd}yQ(EV#a|KOW$%mJZRsX7J78a>)9Lcj3{SEKc)Kg3w?4V9N9W z5ZCGAjW5ndf$Mqd{jUluzCD7aM}=VU_zYO|HwM3**aq!%D>fgLAO|ECSO1i#WLpLA z$a!O14EP&}W`(9OXl({wU(Ucw`z2s?Q<8sf;e3A6%ZXs1K9{(hjmFt`WI(dBfVTN& z;<1)oq8eUBH`!!R^`hlOO)nnPuCAeNSxexv^IjV6Fa=|0p2NGh=MV>BA^2iF0ykUJ z@kU}2UAZU|8vQddc1tA?pEJ0H%T!-gRfMWF4B2U%%gQx1<5#y5__Or`rA|k%MOz+> zR}v!d(SU6iU!bdsAKHBCptn5_qk>o%tV}7Rmb`Le^>=`bYgT~!6Qjte%rutoyAFRI z+0FR(m_oF@F6g%^LR!yNW=jFf<+sD}UbPU-Syf9^EeZAqNP%IW2_4$uftJOR{79?0 zC=sy(O<5aIH00BeWE%+m=z|gCHqw{B?ciF2C*10KKvwBLqpBSn(8(~2C zz2_SI$xO$mWFB}77~mbq+C3`MG*56McuN;dm83kk)FP%wSBZmggHPE<9 zA1u53Y1;g`Frw#(SME&))qkhS4i6*J9ekDi-9MF{y;4Q?H=YDz=^V`Q{tAZiE$oOy zIlUtHmr-%}$`)?ggKw5!A>SU2$1lRCF>S&Weqe?$`6F@@KFySXgNKK!YxYfngew7b zy@d&!emaCd_UV(X`+KRKYd#8*BLt(gJq!jpd=OFg+B-pz0 zDS7eh6Em`JjJNN%9^Czy4Mu&7f$Y2sjxhsFbay*^SXhcLgaYu>LTTdXv>jhN*`dD_ zH(z9F2c$Sp;cptfh>LZ)Fv6{akyrjp63S%ILbZ*CO?XU4&Hb>xYA=*5@de`@^Qimm za8#LD%3FHnF%Iq&pk$^g4fv&j3m^Hym0nkxy)p#*Mb)WnrWQFa)dZqtO&mYe3VJ8% z@lyS7xYBr#I^NEpNnOGCl8A%DpbaB(eF5B?L~)PW6bw~gLo#1jqH}jL^6$UK+XoI~ zcKSb@XR`{Vl8fQ5pcr#MavfBCq1c-_8O>{xu_aLrx98qQj{$91_-dTs`o}!h!eW@p zdg@_Lmo+zQh``$mI)duk0gN`wF&FX6sh*7j24wi7o6o@n3+5;nLH7W#5otow;%b9>y7%4tH@!6rN~+7=y%2+efM+_t{RmdIbB8PbU9-#Gm-WDk6_!xVrKK@ zRGM`DCsY654WwM>kcLqsyQ5<##P88Ty={}ZS$4wcRJs+frH3-|J1&qKF(LSw;sq<- zX7F|fEyOeZ+4y?hEo!I}g2fWIaC2${q+Weag0{be$`$*F*{ZFW=U;+e;dYGQh7JLAF|KmicnHvtkoiOa9>av@4`| zQz#B}dBMuQOmJ1y!6~`v?AxA;IFNb>4jWCw--Mex#&u&awfsin*VWUuhoew#CQ2KA z97U^vUsOoO30y)Qv1Fb#h4NT(XO%Q}Pw&u9iyXR`UWL)Oi>US5f8fdQgLBJmF>c*M zI-bifpPK3c*Us&y_iwC$^5xu&1DCbT&C<=VX!RYMqkRM2@@G?fqZp7Fs6hUdJ7C%A z42#Yx!{eq<6#LLXA96j6y+0q~gC)E0kVr7{tUnSjr!;nBRW=b{k;1OtlZFlkpSg3$ zMX20=iE;VbgYva$RPn?x9=y~+W_NF;uLDKE?wSl(R^KC!f22XqAzSw1NI$4MJEG@G zJsSVc7Q42%!26nQq--4@ZC*(W-gI^lg4~STaTWBvasZ_lqltUOVw^Kh1VUPFprt|( z_1gc3sI9q;hYP~tb9y2WFy8V2hWq@C*|f! zDt$Enp##2JVvZp+0bXdRGJcQLVck}B+^YW^O~s-KRbK&@550zgw%f4%cnQ8wYog;< z?<6H$_Rnak0yYGm#k^-*(earZNw>L2gUJ?fKf!}joA^)@{))0wf0H*qqVcKc86q#X zid?li!}wkk1}fD?cW#=9p;HB9AbUT=+|VN85;5e*IWGVCC<$Y^-)hpDhL04~U{=a3 z*4EnqbH54U${UB_@U$)T!kx_OE$&B{ZR~RFxZgzG^=1%*MLGhnjJvSAA`5ukW|*I^ zgt5yKV3%7Q^!eJu=VuRSWz;*8ACXE{oN&U(FFvR)^BoRKi9zM0d^{`Lg{#xoqj{YgM`66Vl;Fvx5FB0=4jX@Zas5HcpgQ{% zwYECI@l5N;a+?a2m7WQ^-`hC&V{#na?|O`i z`yw%T;}~_?r3j~k-=l2wJ)D#zjE{^B=+u8%xZ*@8ah2Hxug2R_$I_ETW3@IZS#SgI z-%Eh_^h~_7R2uAO$Wy~@Q8-t>0Xh$SCGH9rx+G?n+nw_0`q|B-UhP;LWiR6aD>Z#^;Y6hsM}U z>Tuvb_DI&CDwp%#@nbh=a*l?B1>Z=!Q7o%<+Xas1v79?R2cL#jkUAyBva5sX&>dxs z&e;fKe{MqX*H%yqZiL(^wT!u^B>poEpdo={)OU9-qm;XmTEF^5Z!TU7cMHS>fr*~X zd|ofvaV(xb^KXHcOHNRI_ydBsSd#4H8 z#`l>x-!R0xueZa{I1N0WVg`QOu8{BC%<`PheWtD{m8yFjBe^o~$fYA3*Tt?Pc~o7Wj_+RlhV!;2XfHbmfjhpD%C~B;YiKsS_lrYmMHwpe zq>s$;F(603r%>^2i;4FSb6gf*2bRlD;Loa~Fc!l3>yB5@a|TbKE71V*zA_|LYKX4$ zP=^b3DV#H?x;%dP3^}~A1#b19#Pf{;VthK7{#$j2$rJGd-=iE;sAMLrlvWj-&~(Gk zA7+7DcO87>nw(1u?~|^8b98&(a$@~vEm%$%52A;}V8T6RI<0Oyw`=2Af6?FR=zL|c zco>AS!)wTaoG9w|-U)@uC1J-cJ)X3+5nFn1Dzl@l1I0ug=%T09C~|chURXW_zAA@7 zX1xGAONu~jVF@{YuNJ&(sz~B^1>VA|TS(+(W3ay2kH3P11&hvhGXvt^@%8L_vUrXc zTQ{r-TRx?e1LG25R$C+y8GVO;EV8lw~afiRbPhcm-BJYi889ys47TXxf-)FE|X=c$1rkuJa(#zkfg;)q`#M7jryB32>aCi3TYF2tjdR#H++uK>%4h zZ9e{ze2X=k$J4c3#;wSB93(CKh!)EEXkRG{N@wHXkFqfKaT#2bmUyyg5y#&n-$B0B zoMZ5v!SU4%AnR0%A{!;ZRG}YL+Cy>DrV*ToRqI|1PLcYQZ|+{kTN&E=lFy$K6a2+xK_Ex~JY;H){tm$_{5A zv=^fsvjT+IRe;q{4iOtW4~y4$(VpTC?(_U4&%Qg8aWVxQgVm0T@MP#zEjK7A+QKwS zcfgO&he7DbH9XSz0r%}mguiQz=({cYxI#RKG0BJz zAAy&-c}V^OBUskOfVKQ&qTZkaXAIRDpX&dxF7YRJJrTlE&I$VWS`GE}F9VX7DDC5Yie-fY-(zz<$Lg$QdDEtlf;@Fty6i*w?1jrfCQKK<4H#oS=Q3M+-i+RSaSDunaewbX4C(5) z23y|NPg)t)c zDTXYNkVk(DcO31sMZTvc%!&wyG;Y=`<&X*Mlw!&1KbElbi9d);DhCPmNN8+Pfc@DG zwD!v%x+9eHX4>yY#&;V$cs0n1*-U|PlDEhM@n)uSQvw9y6nMU!>msf?%N*QmiRmgE zaBEx+-J?CiNY1}va?cx@%en>H&FjN9@+>T4uC{-8^)aLgn}uCrM^RSVyG-$a}23WhuUU_i!E zkk*>SrhFC0bN+hV@0JRTbXcL%NA7;_zYhKfrUu>eNv%SNsndH5z zV7YV#%0%5q*N%m#dVV&2-NWUIpO%t01AtM}0?_u&Z!&#z1BAbq=T*r?p}No&m~NYZ zibwNVx4n@x=?|9|(XJjzm&?4=!oQ&`@gNvodo!F|SYxXSwvra0E2a-9eGxyYfh z4wqql8ctPvn{a+lIcP*gF%!m0G0gV|VOCtkkT>G=MCK-3pTqFFfQM_ww=pJ-{iu2W zAKA*GhGm+9X_Lzd;^7>{{GG?;$v#fOQ8G#3W>|~c{AC33g&MGSo;QTfS%?Qhvx(*F zc(l$cpc9uI!pg{dG~$CQZ`QGK zWHt(f$MaIB%R-8O7S5S623zEx)B7FfShn~s%H0pc2S01*{bXyrb5}M#yEsA8(*IXo+I1vE1{!6@UG@mae#GXI|my7l>!kIye+ zL)vsOSkVAAXN*C@%Ng!$mBVuPE67GafscJh(MNqIY*+CH?=69JZ?_xqojVafo(qGa z^Yh`f(__+dN0_%oe~cd98p^Jmos7?pAA)UV!!U2{v$tR-!dNSM=6GtAY7%Pz}~6%w=v_6tT(CCzyYQs)FnOH|d~yHT4_ygSp|i zr~!BW8-H;T%n8fFGk$WMKV21mC<%dW@<%#gRs~ifEhrS94m+a+L}S>Nbe>rQ$Id%a z@gu{SlBAC_F6)w#eWG}GN*o=LHy|68uHpV}Z>YVMOI2w$yP(y&ytIBjY`Qm5P-2t{ z38ASFq?nJ6m&frmrwKt-&qqe(dlPl-+JP~&nB*s&N4w9I&X#$AJ;@Vx29R8afLc zqr1>*;4VlQ{bIVg`IiCRh^ZOPjG;;#K3h0QpXqQ+v8=E7JGhkeo$n_B105J!`W5ut zW0;tC>mkSDH~r&n2fqA3@~WNTZ5(5%*2g}^>ggoGK;B&3A|oM4RB0koNzE)TEtyW9 zDo@^*|6+vpx-qZZ#*tkOmQ3y=hJYbwdDf0kr3w}*+%o=7GBR#U3Nx!&__Xv;w*;8jh9vK@EGFaIM@ zc}tr2wYQ5n1|I_^FBl9rT2<%Lmr=MVDp*&zP$ z=e!B_@g~X>cHr87_n24r=KvG#P5&k8VOP~!a{O!)S>ecs1OHkO^)-31(=!NluSBms z%Ycy#gp>e|U!E9B{Y{Rc`Ge&)I^@&qPv2>nKTViNFDV6pB|H za#=cg?(>F0IsJe>(n7pLpYMZiS1t~X-DBqM8m2p^YvAF#zZjW)FY)p6QF<`B1l;t^ zOrG~?;x~RYo!c9Z>wZY^3@$H4vt8B1`r0_iTQNY4w+iD;y%coRu!4?(|44k#bxc^j z8E%?HK=AQq_~avl5?oeKb-_%GwtvnXytM@+_}k#wxzi?o9>Lf@xC=Dm{2}&jA>36s zOa~_YKb>KQ_f8QPJ8Z}7Yn!OdK7#${ zSE07b7kHc#K$2U-VSR}SEUajS?X7<(Srd)&Ke?Sqa1Ja!la1eRm19dnGJR7vOdaA@ zW6pkr)J~xiL21(hclcXvw^|U zcwG6r3)<2+XYS_)nArFll}0w8mfU#3>s!aLT2+@R>-|E=f7W2wI*T+9Rg&T`E8J=A z18Y}|=QmwgfTFuy>3>3q%9~>GrI0hOsjTAqAvLgz^~R#0lMv+c5F##X;|=b2XF+8P zsGAA%re3fi>63DyOC%USF7O3aqnBvenTao&!(iP*alyv8Bydf1#Tr!sZIP)Z>Z<O`inML4SS5L2?7#`{T%{fXrk73Fs!uul&m7FhcqCy~Ud@Vt8m;a=5 zdASg(p3bh|T>X%VWH(cVz9KF=4=yV=Nla)Pc&-T z{X!1ZPh|^A$}EO3r7XCUT8_(4l@evOHJ}@Gl?i+Fnw41}0OGE5z~hW8$Buf1$NF!h z)MLc%bu+;uz@cnM?i%u;aS=hQkBqUhJgRG0VWV~<6*;&~Zw1i8ZP zxThwoR}|sb*#%U4U@^4Qh9j~(ss<8s&f7I-WZ zjR(Vn3>mU~&niDY1H~?TxSU!x{MGcq>AyqyI}{QjqCAHrD9*#?+1xYbyO8IwGXQiJ zc%aJa6FA*H5M37=Qsom9a8uxB!Twi;^mdsz&hD4O>pIF{BIg69`|m(fQVor$+>egS z?LfDE0;x#84ZqCB*oB#!Y2o)?rgMZag-K4(GhYBlmSkeiOCylc3Lr5tCNQ(L5`;@O z;VUs|f$i;|#E;)X=6sbVyR}8Jq<0siJD~(~UyP8sHy@B4{#Ilyw@;V(Zw+1w3ql8l z0`}L^JMggMCY?H^5+!a0aLk?AoD1+Z=I5o*y^AKo?;3I5@Xv?j{%RAZGmwW%_uR#S z1*`FgY8-C(bdo&V+k`6(Ea{A%XzJ3pmS%c{GJ5P5_@*zwYI6a;@tF<|q5I*z$`D!n zK@-oIE#??8M{$?B7VUAa!0$hY$erb4SZT2g_WLKGy^Alk@k%E{bThsUTY@`t>`AHL z6|yd^lUb&>j=Zpcz$OJRMxpiT^uS;;J(?Ipo(}KB;TMa+g3e>ylee?RE-ZL!-T*&S zZ_tvxOKj<$fx9kGXNwDBqV&ieVI<-^gxP{iLG-0fGwdE8h+jm6 zp=KzVSk;?Tmlx*rllTW_=?o(;jFG*oE7DqtSd=m?!$h06&g<3iw43(K+fD}!KY{58pRVkGLWHLtg@|j>tWR_}JXY$Y3$7^R zv*=W~W~GR~D@5QP_byrb+=?8HT?_Fo-gwx}1RmeK&H3mjVa&~w4WfT&6OvgJ!Lp2qs+z!MrON z5)BA?_uYUg8YaXjG!`DOTf#fc;aMdTk?>n-alj{xY=8&0tJsiGut_Y4$5x zA^T|*C9>5pnYD!+32Aszb%B_Qz9AcDhTzl_^)R7430gC<$i2N2ahkCxRXXK|?T0y1 z#^gir%lAKAcH$~6lW_vcxpk0TqYN=4Tah(e2!~IGP=?q+&tMe1_BZ5Rdwm@rY7cOo zT_?222*atf{-Vmxi^#v4jcRj!;8EZJ?d~qZZjn+__MfoefctqeG1HV|AIU@2sbZ`y z&VwfceR%zC9~KPqNb;Hrd^YtCUihqvI~={i#&IEU{xt&>t}N%e?7no2o1-^4w4xYH zg=H_^k(%7O=riLBopsHc9sA)-c}AU6Q|v zW2}mr;p&LzSUmMJR(CH4H>+EaBlM3{*dN6ju0JLcaTuprZ=^d8D&s@e3{#dGL*Qg{ zykVk;jaw+4r{RQCuXIA%1w9xsm4rQhm6Ri)pqFV3Uqxp*{1tjeZME*g1I?9iq(BG? z`ejk>${~D|HGn4St+4!x6-<2b3}U&R$!#Sch`VmVWqMaoZM|DiMUTRDtE1#kaU*HD zd=oor>Y>Or7i=YO;_i$f6h7Pqx-Y`vi2QT1a91)azpbaHDNC_3>mt-&_Q#$3r(vw! za;PUkxO1_p;6~U=lr}fSVdFZSYv_vaCg0}r9=BkUWC9$B&&Cy|P0)I$iXKTa#V6yQ z;aa&`QUWZY! z>C~`j9Ne2^2sYdqS`#PWDMcN+>wydZPt0b%sBSU3AIvpj_ecozm<#l~;to`Hl4l-j zHlf)%9iH9|ZODCDiZcbALtNMs6I&KSpHSN0_I6 zGf=A_lk=!|K>Sg{-j23}CmquSb0%Eimp>Om)j7@dauFlYjG2miTbJWH#{yuhm9f^+ z2TBwdup4X+V{eECq|N;1%XMb1Kydl2o|3|33onI z+&d!-*ZrGMuRc~}wh((@1p`D)evlO8JcI9FWXU$C5V#~X9ab16!c@QSFrhV(-W<`O zPkGYxgN6@$an30_6dpp;_9TPjnaf~ty_~(bCl)_9hp}IUnn}PyUtHH1MjSqCV~)rl zD%&vGnFo41@VR1ed$>)G^nBUowGM=u(2e7=O$xR-Mk8ggCBqdmH4 z-{yqoOQwR&k6mz^yA}s?CV?u}kujEgC!tb9*PhFu zP#i%EUp#`dL(;rLuPWT^DaN^xdDuMf4nI6>Bc3&%347zsaC$&IjaHMQ?{loFmm^EY zJ4{70`&7o@@kBPNR0rSm_~OHe)#$j<0wZ32qzaBn(B9e4s;+oT3|;bJ)}&C8q$nW3W`8U!UWVm;hot8Ol z@Mi`q{?=1JE_2bJ@d~Q+!|>}d0kvp9%)hs6Aunm=7R)NOq`oB+(0A8l!QMYT)J@(W zY;#9&TIM}E_*n{{svCh+=w+N;djZ>}HE~+LBCdOE%g$SO6&^gj!u&|Migz5h5Jh_# zsE9RzQfCR?a{E${{bdHab0uJv-~mQBM}X)_3yi(&&-m;pg5>@&(4NiZ1Do>DW!fLc z<%}*EWMlw^;CqwvJHLy=AMu;W-|8=9 z$%#L>bZR(#98d+)AEU7HWGu`Iog+A3>j&oc`FN>VMDRc@mtC#?7Z(?Af$S#ko#xt3 z^5RJhUP}u`ow%(io;S*Dym=n)J3NQU73%oe(3&U=aV{*2f1r{qL`3^_1x7L1@aT2~ zjwY=l2gU92;j#k8+xQ|~zoHDT?GwUa6>c{AN?LHzqMVFeR0L)1X_(b&fS<}c$$}UL zKi=<#I_G{I_$Cd3#(q#&qepwIePN{i60GwQ7X&q%V-~lY(pjQ{uWil~(frpOTj?*u zXc{uV%u}H1dNeAhgwr*r!a>(18UOQm49{L&!QVPMnAhU~8s9BRYOf!j{1ibAtj!qP zy<5=wM;}_Hh`{&4FO1{iZ{$b1JTP6~$@0|m(C4jB8wVwErfdiM=-D}@!}}V`Mn%yQ zVPPzMC;@x#YvIn4V>IEamr3tSdHh%}3P;MDsLQ23x^-JOZ8x3C{NeIfS68y!Or#2o z_HulfKrc8X`IT&vS(7FLhC| zY9@BcCStXy0VuT(k=cJK$5mMgr@lmD#KueT?1~imyJkB+pDN5N^=_ux9RjBNPZw1Y zi1Na#TItO#d0_TsF5n3XHnQRqv%vigW4U z-=8wu#WLt@qjcbpZbm;TTlzHR1UmPp6Z7lcFyFinSNK?hp=c2dNwZ|z1a)HnyoWB- zegVWJi-_;;B)<-Q#&-TN4PW(`hRK}cvMYQ#Pb?Ru@{I9pvkIyiU7}96=0f!!6(WHWVIK!K8(6NM$L_Z)uj*d6!R;dK1 zIpYMjbJB^>^qZLJuSZ|~(u4N*%jqQE8Tdyp5gm?Uc5ID1RB^L8u6Bm;ZtFXRA?TkfRk$GxCw`41{wKrej~h+!(^wR( zyWQ}eAfDc~*bW=cDDlR0u3*m2GRzcO0sSg^WJu^Ub0Tpy`gmW*_h|&&bcWFin#f?u zKbU)YjLuFv4|$XRGIOi?$g#o(#_E?f)p=8j4N?&(R1<=+?KN=Tj&opC{UEn3JWL|? zUm*LZ8ViIPrh)j-2*imA!+ekVm=#>lSR5&44tZ5$hJF#)>4ngz=HswBwVZ}`*+7I$ z6|1v^kE_F)@%z+&M1QvqH=CGF4um$5DZkC>jFxX?x5^4kd7nul&&YuK-j~ohIz`}G zR)~QYFW{%)WMEH}gXouQ00t&F_>0SM&j|WT zB1W@xqiC?%5^!2*i#qzN6KZb`)a zutU^6T?P{NX41Jn%J^fe7R*W5iXpn0bmBjIc+~vCMCyP0W|}3Fx13hRoD^g`)TXp06r^`rVh{^q&!sNSPDv3#GBTS?y-Qse9Ub5<^^|>Z=OO>Qx*N4^Ynwd7_SoI?`e?1jG zt9--{!dAFv&PMvzSDL4}-vSf3KK0F8varcmm18K~VSOT&f@6&(*Ncw8^eLCc#_Hc!;G6`;OkwZmhPUbvGOF-u7M{LSFLl@a-3mjHyP_sv|c*01PfKU?7aR+))!jKGh39x}GOjVL~8 zgRMgx%RY}KWe|*ZqeqF+t7>-Jg%{YujN&%;SlE|81^8RfV#4gz(7Q2&^ql2#ey^so zHxg%%>^YgF;QkFTsI&pTh9>;m*-Ss{6_Ls}QuOSeizxOriBZz4C*Qs&Qr~D7v`$-3 z=ciV{_FFnM>aaF4A{ltuDV%hQ9i_J<`l-Y8aUgn6oiIyAiIY=0%tsmO?5jr_gk16I z&vIHW_(JDuY@xZw0j_hMm=Bf)RO0qbh<)XX-3oEUVuPsapL;|EqVN4+{&X*zcc_`JNLzsAt#{y&hyq#Y>4aBPBA{G) z3(VM*LJq8sN6+X-bi?#CD0-2Lo5XTS%gY{Y-E)+@bK_yD*Hb7de2(QyW1;F(EiQZ% zjO?;P>UBW`Z533Y+U!4A^IM#}_kYN@d!xX!7hOs1x9O4&XA{`4e-vuhKcuQFP3YE1 z$Jk(A1bsd~n!4({!J;S0fR`F^y>$(>`?bPkZ0&zo8F3HE+z^DmAl$fr1XB4j*f!`x z3oTa*PMBXIz0<;R!)iGYYTJjq%9hgMi)YEzd{JI@_XQffnvaV1S!{s$J~Fvr3%ELl zk#KoGxY!_p`>#jybAG5o{w*cQ<+${#XSl+QXWxj8_d&Q4xf*sf&V(aFAt-c>#r^NH zu~eoAyKXg8R}K%8rZBcjoP+`L;O_vv@y@Kh)P!bX#M9PNQ&pJYgSsS4)F1_FP=G#Cy&h69mD zut(npO$wqhB(IR<+NNUS$1%EPPXpcBqRuuBNWrU{C6K#~2&!d@7&nQ3CKg-Q!Jp_Z z)}crT8>>ZdWqMKBkw2=Ccf^`?GEN{9zg^_ARub51x{*60bNTxEX&~{;)iOSF^t@kz+fSr0i;}K#`G;&0ktK%DasoJ~+I(1=+(4}734zq|9Jr!< zfpaFlCO`GJ;O3s+beH>oC>$&ge&KEqZlRCoXNR)?$#2D{!<%SzbU&8mFqM8+3&bn;E@4<-1amig z8kBUUF?FwzDu3jh%xT^z%QO`nV3wRN@n}7r!<6x zJmJNd99@lGcQlFQjGYuc&(bpZ%Ka8@2jA3#uu6RaJpNfg)EedK!v%U!EniRf7OLXH zMY))m+rZrG_2!s|v!GFXHiZ4V}~y#%D&FOj>gn?X`@ z1v8-G5B!y4bcVbGDf#CG4apOMHdNz6bAP;K7D#5PRHLU^DvXXQ@WyurLDGyn*bwy} zQQ;g2wQ)1iRZ$gI_Dc!uX5~YIJ;#`QdzSB^G74I6L&=eOHSp!G4D1cIVrK_Whx;FF zP~v7RiL{>&2S&&EcRYg2$=GbT_;P4sxgU$pXa(5cMt<&1`x4RET}aTf zei)ymk7Cr0dS>+}FQlVeF0D3en* z$xNB!6H?VWo-xrgg}0l3l3;Gnvi)5h-fw$kvMz2rINJq6{(u-R`ZWiiqXp*k_tD$R z<-Q*0;Az?-?EJSK>kV#VO@1wwUzX;4Dh<%BzlI!T#qn2wx8#!6=Kz zApdz09(PjX-G16dZ*9E+sar09(qkd?)IGx%`VSFSZ53>9lf;#Bd+6?$lX)?9B5*5j z25e*}3vNG5BWADX(y|-v#P?4ohQwC40H0Ed zvB0UF^d8zmX2~5wCDR``GT{-FU#^9*4{6*itB!^bsgTfwEJ}XOgp2g9? zb!H~r6vlPT4$Vf#ZXUgzE)M0^rd;3A7k;M9=G!l_ftQ8O=>AxZV^;;yFEiJ$Q98rK z=14IZ#Q&hsd#(ZVtw;J0}PZ%rL|_4Q({66#`zN9M3W$p6|43D>}{6!jrC6sHGTZ67{Ns z3`b^yb@vM3-|?k>#qMZ$*+I||J&R|x;1S*0Tmo6gQ*or>D+&4ShsVF3!LaDx_@sRv zWVN5gjEAAn@?8l=n#br#$#Wc+_bhoyco;pN^RwT%K(6w2*?XTw1sV0Ts6oUxxZV

Mo-f-s_FuNxhVp4If#kNox>*=?U&+Nj5^G8Wk#as4` z%QXDt!gAiN0P>Zask)wG=y#ptR4^eD*ZL{J`Q_#1C$*=*e-C@%O<)EJiytNHe3paO z-zRjRv?ecSVHE%RjASaT9L}!2dX70O^9m2=pJf&ooFr+@8my6-jzIaQh#`EW|#4%-fd;4y_VG}i`F-Kq|g!xX_JsEVu;+`$2H zf~O=0@$cO=*uB^d+6J0ICsGTiwa7Cf6|uP0J01I}H1BxDR^qu%kw$-20$rSl4;o{* z4*y-)Z2AjhbtIYQ%6WLP{tLO-qe|z^9)+C>0?gt1*+;F7$={hPaG8%h56Uj1UXeS- zcYR{YHfnL6&0?ar{{zh&{YxJ`ep~LHqYNeQ!clsBHNBYHg-0UKL+badoL|-j6UwvU z;kJ5uF!}&2_uow0xwGY0>sDGc=#SRPhfxAOxMy`bd~%YcGlaMtN7)n-zaogdTy=>F z*jx)9FB)mBPXJN<>;tDoV!(2cz`T+!m^eEZCJRE~_LeFlu#CmQ4+Xe%jvnr={K^eq zAHd?;72vVX7Xx~7LF~K{Yj{=`+qR8Et(teRMVXgac!Bv`wP(@P!ME% zFMynXFX)BGb>y*3CQ3fF#`-y%;l0od_LJFFdef%}LMM#lCGAeagZdThh1C(T@v0*f zyf8thoMbrmH3aHjJOMvdOVX-#8&}@cH<{@R;%L41#7U?>@VwdLx=ME-;)D} zy@|Gq1unXI6+iEGg&&VZ;k2PXES(z!ub+A16vtR>JGT|~-w)yk9H=AT%0uwN0bNXe z)_^CvO;Bk2L^g^0?pf{~MsKX-?hReSanDC+#@)#RZ(~QY*0vfl4{*NHByC)`b`{id z``HJTQ{Z0Seaf7Phsrof8tQTcce|I7boM^jmn36o+;I%LI-b|Sxs$S5{ph1}*`WGi z3wDSMp{z_E{Ss;f_lD%r%|!yHeRzP2x##p+odGThFNATwYrs}~E1HiRX8t{yj~WAi z$=`Za!cWs9-#2KHN4pKMRmB57a7+h@0@iM`OgBqCpJPEYzVsKN!boz41CVV=! zlRT7-g2Xjtq@3T!c-p?D$0DbL>X}0LUDiY@W!-TCFOYBsKC&{fpLQl(r`406pngII z8**%=Koi!{cKk|K%5k0aZ543);9|5D9R{Tmj#aom6Q5;E3I<+1Ab*q6aQu_2Sl1#0 zQAsmFO5q$mO}-EFnmiZ-=yfFkxObMysIzcVML9l3Q8+ctyBrdNju)-pU$}jGt;Z+sT z?)ido*=Y^;59s0YAsOuSxeLWq0oL`b;hwE1V0lG!>(xYNvH^)U>kKP_btTyK%OgEDQ zhk+G9H(rGFkE((YpGn+|=prasxBz7T~5cVQ3lXe z5e~jw51~v{i7gZCg|eVSXtOm4b}9vehd~}_^{&B=$~&}Z<4w4x&pCFkAHo2R)1#Wp zvALGXF(2jn;OC$*+!}9zA#28iMX>>18R0`KT|%OjU($pVlfikfJOsT}6*M=RVmBPc zGkgcMwP_?DeZ>VW-%=rBxCo}G{DK`1mFY%HFUWaxjW)-p<6U_v-X+xu%qFYVWS*8Y zRhCMHxTrof;&v0#Rzk2`G5}{SNWnO*7L=PM1ov~58SCLEOzWrJ_*g2IoLG8`bv1Rs z^&0beeX_e@{CjiGEfB}r{`11Yp%^e|;6B@?TQK2pFnPA?F`1aC%4LA}kuil-aG87u z&z^iv?(UPowkNNNl=BWaeApC{6xHCUWI6nP(T-1EDllH(lK2+GV%*-PjIuZ0>eWbyvHaO$f0?K9AuuF2SMT4SbJ6( zx{D-HCuo0Cl^yI7LunKYP^HJX~cg)Dg9)<0e*cMZgY_>mt_V&*V9teW#{3JC(Tr}OeZZ--Cj1`s4Cpr7^ypj}J>?&uGt2f8>PV1Epn+vcIXYb981 zvx4{cZsU?z4Kx$xW;S{ur2F|Nx+Gjd@T8dI)m)C@yKxzL?X2-+taO0Rde}`JT<2iy zI}31n?+U-R?<38^r%BFQcla}zz(+M{Vr;#ECj8Wac#QQ-x6r^ze(21H{$S&S~wi@ z8Lr=*DCk(e8}mD^(frk%3v03pGuciAyS-LG#(i~E;J-oLWR~-GRpWMhKVsD$3MXew zh5Z9hh_u^b5O?VSIX6wnbjW2^2Ht^;&|D%Rn@Nq$g3-5khTx!4B|Wb%g^lf-jAz?w z69vN#2uQz&GS1@YWmZAtP2AATG=^FRJHVIHS#YD?26yMZK^hYPcRr@Wl7EAsr|(Bc zB6&ExErvaqc7b2a`wLgyk70|fChXXw!A8WtVU~g!USI>D;?GGK=)Q>Q!HZCk=!QPa zpV0|D#iVOMTG0PZfKO^dAzk(RY8mBnNM%=<%WsSnKB=S*M)$> zE;HN_*9j(TLU5T~D68Eh%ZmwJg#KC0bjBhJFrOg98$KNheZ#$Uk7F?PCgn6&?hbr+ z)i0~v>_lD)PvU8&1GzLVo}@XA;~md77sv|+_{o@pmq~CpYdQPR+?<;!@4>-5arnc%bA66D1Wue6YFt4W7%wQI{cZg?sCovarR}hh zeTSZhddZ$Iaj9(Hl&Rx*pN_su^_s zW=(eBt|&XImQS*J@^7 z*Zv^EGe6+YwjM}Hn#9{+*-4K7)FV!-b;)%1I9zA*8%!mX={nN@Xjb-u6`>hu)ju6; z?z*D~*RdaWwSk`|V+!}f{GsW%qH&&*HaSqpF@vlUacOS=?c=fqf4_c418r5ole$&J ze?=5+dU+WR?;OVYof;%g>H}%_jf1v*T#rCRmGTqfSXZ$Dy3*GYTIP!gZ0fB*Nv8!r z&6lAL9LJ!iB8Ik&je{M_duYkvGNy9XcKZ6-HFOT{CtB|~Kd+0j;4t@0cV)gL3TuF9 z&zui`rFu~;EQ6}4$)V7(NqD|hgqO{%B>V^o{P$xrY}^q}6u69+LqjQewE*WkaPGMvr6;C6a{uV&#QE+>+(eFDcD zZzCGc;=I_vFe0Y=h~x&O6UTEGVS(Z$I_>);zUq{JtZ*$4E*|Jdn=32v>v~y?IVXfN zm%GRjyBoByA`qI>`CUhPaG=y=wp(C0h~=y6mg_MG zbi_!4=OnZ&Qo@W~OBwpwn%a(whdld5@XS&db)Hz_j05qw@>{;7Id(*0^#h1ob4)m49?$&Lme{9rKMy+LRLEz(jDJ z#lwj8v$mI#6Y?U1u@JNR4?`?Fklq*?Xau)MQJe^z^g-fzK$s^t?7_mPLarK+1 zTk|tIRJRXgHV|;MTn_<{&LDpFW6nOXgMqVkaO8<72rt-#S0*%(dAb?`v!QWtz~`~a z37zxcW!uI~))2tWoloJ}2)E~)?8BalN}`jkt?-uTAhDk~Nw6h;GPRtN1OKb*%%iCa zzqoJA6q%B_P)SLHM4V?|lr$Ss(LgE9G*F^a=6OyegeWSKA=7#GrII9+ib@ovq70Ry z`91G%t#?@OyMF)p>; z?7Pj{!wC3(>}6-abt83t@6C1$hGEaiJi6cU35nGd<4yj_aq3E~&3@|nkneWu89Qgr z7y9%gxMU9y-x68gYvt`QjcIfQ!_UB>6%O<{JgIY^dQG3`+mBuT{_ z!iIA3@M;$b^D~BVOIoSWiPu$ozMsY3zs4BzHv!h1OLNF|pCQ<-mx3=<)Hn{u8k(ZsL3n#RFj&e6YTcXZ$Ur_7 zb6W|s*Jb0;W#j3BNHHR_GmtS74n)_FZD?$%hgOX-)V|{$Oxp3A+COtZ`M|%->sU8d zhj)f-*q9C;+DDj+qMwN0F(*74I1dg;t${CZ<7i#3GG?jnAew!?w0md@HX4ZWcTG(L z4f%N3)GN$8DtVRol+B|mAK&9`uD>v7_?9#aohQ1=1#tXL4vb97ChZFvz-)asxt1~= zqNA3f3O5hw@?jBN&r8Q+8wDY4FC{sHvmk%M9t^hnK+-h|$iu_MC?PyXzBWqX&uVFW z*1jJ*_LWj4RUVkOx`D%^?YLrAAc?bI2-lZP;t3nqYNFZjd{@mQ*4JOODm9hIM2dil~&RTjK z8hc|I-*9JS9vYIBGXWY^E3xk$k38eN@)k>8GU?0jacrCzIC8)l9?sOj)^G_d{CW*V zJs06V)nBYu>mHPdJq!FnK`_{y0=K`(z`pYHxJl<8h}Evd1vbL4+VUPAdedlDo4_&l zLPSZ^mLeSTd{3o^ir{stAy~APlgQ*;D%q>bd$Q>O9$TWp?XPAdSE^^{shBXHEo}As zQ+{Axr-yf6PA8wfrqY>TqajITIoGTffH+@S-rz(DEb>(%>18I|EXy>Y4MP0qHlJw9 z9Dlf;PzQDz7UTShFP=`5Ty!CQ;0=2; zM-pBKQ98EY2JfcTq4ReEkd2VQ*D)*+E>J|Lj7sS8IfItp&O+dpOT=qs3lT2*z%Eqj zrkjuL!qI?Iv@Dv3=c<*-cu6k9zrGRjUz!qs*hTY`L-6*xo#?XrGsF*fg6j{CM`?8p zywg|U&K4Iiog~5^6bpe_97E~^*Go6bOfth9F*1}k3ycn@G5HCjXtFp1?EC`J?QbE> z;@MK^ox-@e><<~Mwub=;&g&R(337&==xm!dq#V=Nx%2~B>%H*3N~Xso&@a87=;dx*YQ4;8rRd{h0nfa1e=*DLlB@jZVA0X|K>! zoX+u%4h~p=()~P;h+IOqF1d);gJS5?8{s5sZ8@lLvx6Pn?_4f)VMn3~daG@Ped~9? z&kb8Jr}`yfe&1#qv$_8}S1*II@hW_hJQMPj(l95kjmf{9!=_xi28&nqv%(TVcsw!( zqskpH&Xe0KAKXDA+bLW<`>T4-tYBO^r+mqyd6G;0)&X-=*2RX_1uxEP~ZEcuK z)m|Jz>Bp*QcU2oQyVY=!QaYOb7^I4#AGnO#X8NY03*W4&AVvv1IO3jzMO>zpswIQC z^||V!4$a`VA_ZsM3k1b+t8l|#!XDjLQ9b$B3a~#YinlV_aM6KtMDkrcRHn=Wlbi36 zU)2F$ubPq$I)V3c^gBDZ)dS9Zk1!7>exO5Ap7<_RoAzr@!tpMM?j>nt|Htc)ls_LA z^Dbk!R4^_wFQu<+~rExGW9*( zenJYB_e+4(xEQc^YG$85E-*9NWx%udROE|q_9uSdZ$oTh3f-!riO%}mOgAHi?e7GM zUy(3un>i6$-Tz>EMi5>pWy!c*@i1d>1XBM=LQ}Uj753W;zs2{#=e9Ih?l>7E=d6WO za|0PxxB%v^T!jZWbM;Q4ILscEpr0p(!_;vP$wrGFoNzD*7P`ga%a3E2n{t;ZcSz&a zL)tJPD9n443%EhI1)DV*u*a+&{0FYUZI9_7DR`Orc|sBEdp{7rv0mothbAma8p35Y zKmwdg**KegIPTAbx_!^sq|P2^s#io`YL9jRhi{43re@Y{~>AQ@(#ZF?%1_Az&6^&5Z8p@Wsc2g<2^LS2Q zf-mSQ!o_A9;oCp}_6)70*saJiLk!+N^B8W;c}|S?pM`QA1}}f80M*GzB!b>?{dpaD zZ=8*PPN-pl=q1{JO_t;i^2nFIB(yA)rE*+v&b3wnWBV_n@}|pXLaQ0#9-LS$`Sc|- zbsoo~_Km=+HrJqVmn`UZ3*)LXEv|-BPdpp;p~mHz_({kZH{Ct~#kTgira73jt862a zMaM%-N+qfe3&Ml1FQ~>EC3q>q$JC`wpmmFLfDed(rCl{ScrBC8IW9=$)-IU z5oK6q9E15!E-;gfCxD_;5F~8WMuTIgv2v0Fgncc?`^+;q{kz_5>LwYK8ad7Nnl)K# z8VDgb1>x3E4AD3HKzCkAz`QR@AmQI>nBK4tT-_0$Wy;VS*T=|Y>Sd~@D9iUfr^Y#@ z17P*Re$%s7f9Zww9PekfJr1o1V0xnEc#lqgMC05@I>>pJL#MpM1qY+py-LQInwSN$ zo9$RQlY`yPpCPIth?UoQN%z?w21l)2bb=`u%GDKlETuRaHe_G z>{3x7Se@TZ=5LtI>so&RFa34F=iY*};m=I|-bsh?S10Gu;Cv#7Ec)T|+n1nvvXDI7 z-ig;Z9>>!_aSSryLJtfrQ0pU#&1uAGeFjsfbQY3M{35dB zS2K;S&*@U}ELJHY62+#MLe73il2(%mYo8q^n$sU*tiK3+_v13WirR41D1_J^@`vEJ zQC!a_g%%|`!W}Oy4y0R874io_PK$eV-Q@KwH|x>rXWm@AvAf2PPSI(DIe3t zr=Ls7FOTc+_J}V;m&B1oNl&oTtH2*eRnW!#5JYwm^x82CXDmpq?)rQKa(~pIX_5zZ z+GWq2+Ehe08Y=Kon~dq|&_D2ebpbiFw4PL$TELA96H(64f`mE-;Zc_t)cW-?xVhL3 zM1#tpf2Ta|5r0cFe8XV=@;Z$F76H-M;;3(O4726wGpP7^2-a;_gclY$6CaH>9BT1r zmq@3fw5$b7G?Zx2c@faad_gSJxV(kU4MH82Fm`7zlHKm8XCaE;42`JMj1zD}`5ZQO zw-Zr26ZY}$H2Uef3pNW$k&+v}ux819M*8$({@2e*;2&3j23udzgF9yveVPtk5)awk zsVc;!>pV<&zXqNjHK6a*hd^_h9i~TI#WPnLL4o5M2@gmS*_DbQDN;+n%wL3Yx>Yzq zd1LiwISJ-z_G$R)Bmmo8%-{!mk1jkU3~_RiOqF2(7z9Z}=x`5p8y|-aW%2ZEa~z2p zw*bG&F2s-;4G0aDC6*ghu(QgXmpP%u^wt7tTyQuMrvEI1O1(f<{$mK|tPjKffHLGA z&w#UsX7fF*jA7#jT@>vcC887jVK7&VcSHU*x|MyQYN~>~i|SA4(ZuNx7+Od^nN>43 z9_N{JW#)9@zgTQ34MD-*Ir!@PSuma~#&26?i+(Recpv1JK?k1?1p~$O-t!2I3brDL za}`jhQWY4~@|9Jps3e9;D$qa2nfmB*jM9ef zbhdaaUVbb9{>({cqRwYlAK#O5cXd<_`-?-0*J12(3JHHROcYa;LA>E7-Bdpd5;{Za za34bKJ!|~yC4?7tT|(yv835Prl6C){!DZ8{?9%6IB(pjW)qJDqp41)SXdA+D_g!#* z^E77iZzJ+l`6<~oXBJlk!m8%=f>6IJ2(nHJuxtO-ke;UkP%=%4cbD6b7nl0usjJgD zSAh-<|MrPo%Qc|Sj^`7_*}us7+zA}7XB{Ri2VCU26qTPzlDQxSO}=)lxSk9O>8(U< znR56t%aOi*n1)^pmV@Tvd(`#IH`sk~KhfBxz>Dm8Mub-b-pbNmbR>my&$isCmVbN~ zo6L;);;Yu-q+iF;qF0hIb(#eSte>MgvF zK04-w;c8ttvP=(@#d65w5m9t9AES;{K~T}K7(cYvfcp*!UX|u~I4QE38N2uyi{?dR zjfNnOI!vc%^o#SsrGSCqO}gUKRg&sq$J%MtFfDR%#A1U#ivRe5`d;!-@>@l;KT!M8=OyD1H1;nH(^Y+ths4)-3ACpR;Gkp!Mnk$04?~ReE<9hMXrw1@j>O4MK zI17d(^tI#VQ9=7Si8IiHy^o+ zrJFwxB@>RX*DVi(@%!W0wQUSD zQPIa>O~?<$chViOYGgL&Xp5#BOWmMg?M>`@{D=4n z)_@JKfz(XVft2n9$Wxc)%YzUjHWnnm z;rR5~#mjUWI#~&@CI>|TcO=L&a`6_QXKfFi`p3cz11AHaa z3YdqF*o!pBV-+50+KcNaM#J!t{pe!P&Hv>-f?Zn$z~Xs4>R9Ho%MKQipI;8cDXE#f zI&4@;r`rqOnQHn_0VV`rN@S`>F`?8=R6zvvlLKZg$L!`-EQ_T zaUE4)&9Fgj71)YDh5Ut*{3E;fLeWKkD5||~raw)AFWlV^+ak}yj1{wRyY(O%)tBHt zz7%h{#zk29eVCeCZG%s<^r(APAuUbil&==9C|hNMFMDIbsCW`|znX-9ArlP85M*XZ zlCOuj=kWyw1wCX?MlulFos}qU3#Ttd)d8@xS!Z)--&(P6kelNaNJjNXRyFhE+uc?5LbN`O=n-)31(zX1|^3 z(e%Ykq(n7HeMPt^u0}dH#*#tVoe;vOu-a-e)%|8pOdrX?xEdM$>pui>>JljZlZ2($ zrQwsqbi5`sovcfZW&#q0_=hvX;92hokt$rw6MS$Lx7JJ{fxoJ8Fy08BD!!nbylP43 z_h$sHBUudv&gXD!56rNN!Gw2F`1srl%vy2|KX@s^fLA_*cPaDhwkop5VS@Z_%Vq4Y z#HaM@gHPnCNdgWnYG&Ap&#GF^dvh+ZCQP@w2(c?3L-N=ONU7WjF@i!k_^1lU7oCG$ zdXtFDKU--3^_lAjZGnl|?@{&jdfYl@37ZmGTIE%XR~l4N{L3a>diO0^etMWY+dpPB z|GlJITd%R~mxaWuM2T+mc}R4m#*_5ZyU5X~)iAev2K>sS*k3mpmZ(s(=HPQ+*Q*V) zzNM25M{-g9F!y;xfzd zpjz|VQAsKkhOeDO+jDsK5m%CA zcl<1Z%gJ3tY>p1N73?Elu5ZTLp{wLISJ(PC{V;OLCmOA&h~DB?>G}Qpss5=JV$q(C zyPo8mdQXd{$8T!$CyiYIhbxPzn^hm!D?Ff?m9wEHb&MF+tY?Nalz3M%ljzBN)9LfA zVa(T{ThPwUbZPdfk^Wu5uvEhz=WG!J1?~*17jqH*>UGdi^>xfguWB;(^aGq~5yfA7 zyy#)&BVb#Si$gz#s@7<849#DQvB8ng%o(_kBle>t{GT06l~jPNz46@K$$RkDPe700 z`RPyIH^;)$LxxT25^Mn3C--?|zuK6u4iot(jQEV9JF2NJv>sZ`Q0-Oq~V`{PqZ zaejZXA8DxHfop8baiu{I+ibm!>Wa!ASh-nnH%B(3MN14<_)9teexCbm|21kx;p4~$tMsw z_Xt?}j05+wFj(o{OxC7bV^sEbyqfnCOK*5U`iT#+T=7|s54()> zogRpNh%xUYSe2lsV3hWudgrtj3}5L73m5JNH+}+Sif`g(7`fh9#&VR((8s2`$%v;k z*ypQ6$j5tY;QN`~w3dGjc{euGr7L|vkvrd$oR9GQR0u8e@P}K)`J^@AGi?>{0rQr( z5M0IierMhS@v$SYsBbClTJf4}j@d&ePx?srtq22|-3l0Sdl%l9R^lZmS746*4qU0X z6?g~kK%Go0T;v6UkADn3oN^ytYt4ce-282=l>pD-Tm;w8$z$?T)6Aa4nDH`Itm()+ zZ`7E~@#NojfmXjSv>9fxp+N;CV!1BJPws;s9p$Etqsb`y=PPql=_h%c9s$381mjS4 z9lJYxK2jr7sL*|n4wn1qr-kovj*mUuoDe}TOICw*;tc+moJb6ITnT~WCc%`o-=OXcVrTA;x_LG}iBmR`LS0@E{) zUfF~D?;PU3cRv^*x`r`JN`ir;)8y&tm1s6%i#HPuh{biTcj~kh!acbeov+6SkS1QjwT2UYogmGl>)#JYx^!N5Zb8z2xso9nAS{fJIs9SRFnLo`dSp zZZtweGznpbcyOud9NyeS$mZrQJ&@cfxXq8FV_-Ci|8t8F0}A9EWYIX08F!Z>J>k0oQFzi6+f7rkk` z8~=@Wqk#$UsPTOO!%N@j6@@%>T&9bKnz5L3@c~_%A50scY0$x2u3)YDoc*gY8P6%_ zQ|p_F^m(WT$Xi5FTATs#d!J)<9Rr5nHsZEv!Ql7pCbSI*Lg=1k_?5eqMh2TOb5b|c z<-%jo96Ouiuy(r6s_rWT#dQ|gpk##ISG{nu!zk^N+|Ojx8Co-{8l~kFjfn#VUaY3_qYpJ^g>t z`-9^2-SqWPKIthcTC4N>^{2z0v|O-mlj7gMsfROXS>x|2_v$k?(HPgZ7!MuSp_ek^ znDp%>DET9XD8=bP)UtD=v-Si^a6QH$yGHWxlMYRh%pn0>w$=BnKlDtT3lnQ*V&adv z{Fk;)pmi>gR8DE6G9UNjw8=R_>AuB5UXz#8v=xOZ4zh}l_+~eKkO{*VR zec)y~4roEEO%c{yuCIQO7eU{B7$)1ZzmY4p3g}VM&ZOdXblgYHwqREga?>qq-*#*hIMz-A8L09 z?+&+@&s_?NN3~Era0TL~Tt>eso)l+A(&(ofaqjwuxJ*JFZ%ME%h`Qs2O%qRuyo^@+#$cGr7uwkr{* zTG?X7VN;Y{&yCu0Y-107Mf%9g8+-OtLi29|cilzk^*_rQPwDemIrk4RI^UbfHH+c|EU`4&`~l^Cu}N*7Ji-YyiyU`a5-Z zig54#RV3o1KG`M^OtxJ&GW(e#g=G(ML%aR!9`ggNfDR7?^P_62hJ^;DcG{d7%WN2CkFWg;yYSZ#WbT1<;0+Q_M#R z8En;Rg!zv~$%)Eg8u(if2ju2Z`?W&wq%w=l#yI`4$Pae?gyCHPUdkg#cdb$u9|QsCZRF zhpu~ZpNkJ(VO!B)aX2y3y0^&tFTuu&?_5s(DqEAV84fKGL+8;V_OP8Zm)UVdIQ^Q; zzOVwdR*#aR#Xkv|%`w~zOW4YPebr+L>(F*>GwD`;L0p`yz&dy**(|pU3$(+K>_j+s zs{;QuPlF^EsJ0r6r1!S7beT>Zl~g-U(tI0m-vwf|uA4fZ zAGObbIDcP6E{({*NX2nX3pc?gNnd(O=Mfg1=4!3ENpNGRo17_PkQPNzZPks`r8gLb zLdxMFH-~-H@(7lvK0s;1erk3y93(Q{!RDR>c*LFQ9aI}hM5;R0KYda)Sv?$M7FR>` z{KXho=0Hp)43Sy?cG3y5W+1s}6WR5hV?XA+`pEU0fZu4a-;Dptjt5JhrK&l^Jgs5_%V(yBLFTNR^px zQvuXOXT#U6JXl$>1r7XM;K|@J{B<%3zrB9WsJU36X3&gaNybtAZbzmaBH z4zsPBE9lT`dCb{U4jUhh!}`Qn?yN4zubVg>Sk?+n(d58xzCqO4u{> z3FSrC)A@3ejLL#J$g@7fA$5dcE;qNoTX#C?Sds!EP08qbMS!n4Q4+dGO{vIgz2=_G@!Vv*Cd!he-dkMXbrZX{x0bF5y^r!8Vj#I{Cd?I_ zz`OQ57(=#9;At-JCF27Y^8bA9U_&2|qMKtLYAn@*x^GwMn~oB8>f~xDbE`Fb@bVAS zUXe=YxNgK*v75;FVJS#!_(+RaDL`DfD!AM|3sQ9fTn2{4-fv0xa(X^J%Z)PgM`x6Eobn{D@k^Ad=lBe$QA~M z?~t>ui(%nyL0F}?2SnQCQMzCO`C%W9pQbRNUsVqRH5U9W&JR%i&;{`NHwdnabXdNV zAUe+0#FEivbU42X*V;JaIEQf7|EeUoO%g`oSq-#hDHm1@<-zxb=A@@;Gwe7rp3Huz zL$mzuQ=y6=h%&!JjKV+Cz~|42r2KariP?!QS(A89hSzb$&87I7A4xpsi1M~?+mG8g z$I-!S64(k?@b2OdjF9SbNYX2UjlJ{u)o(=j+eh=TBcPZT>zspFfkfKo?t~xF9cLIF zB%zP)(cGG2bjPmUXn*b+j2D@TC%4JLc8>;J#qqx_I<`WVuM<7BOdY1>je~v_AIzPW zip%YuLf&|J+MnN#-W!e~pYthC4Sh;J_6U(u-*wnePmn&2Y17`j_Uj6ZM z0+jAI;{RxfK!;2v_|EnD*BlOmTUBzT@VGRjq!&U}KnOXXyccsv*21C#*67x|3RE=g z;pu=4{%}iRKKYlKDXT`p{qp-{QQSBo17$9d>Pi!6PXc_jLl;*a~8oxey-QeE{Ny=kc5# zZpGUo4&;I0I(p@LJQ>qWC%wy9xMItPIrn8@>ah^eDC5K4z9(egh#(R35y4II3qW{y z6J%^^!xyiG@Lly1YWy#OzG!#>Cn{EB;9xR-RMjGK1w2&y6oLi|E|N9n2iW)4Dp(ej zh9_;0LCI4?5E++9E}D%;yBABiJyRug_rAi3W${F~Sr(RbZ-jy1beQ=ov|8<16HZUA zXYH)AA&%pI89jHU?n{1>94>D*6d(YqU*A*JFC%z*YbY@>y$fP-xj^g^;G}^Pb&H*b z@z=#+QFjwrKemE+@7aVq_Dmy^#@ncQ=o9*7#T+>Kw-hezx5Gk_MwoE^E)`2xKr#Jm zSg32t?f;X|wN;9Wo;AW_ZA+kOqbsghybUgL?7yL)b`q}nl^Q?_7Q1y;7LQ z*J$!Ae2h zPnCyCvw9k>UrrwdUxYvV%VD-~i`m&IL-c&%!CVqez&#&6fLdu9Mt(I1^Y#=vS%c#t zMH@06)}{1v>;p1QM-xPE`5=7N!041I*n52vZ|TXK=&&`JzTC>4CAPP4EUmx9YRXew zeEJNH{LO=~oNlI|uYo+;c%LlYbd~M?wU{0%n~M92Y+&^u=RI$;1S_X}Xcr14(Po@` z%47#Da2jB}IgWPG&~yw|SHYg1H}rnuZCHE760(%nVbq&ac)qO$w0w)XVuSkRY zzX5P^Rl_ua9B5VHd`S^;_-e8foKl)Vx7aF!e(7^&>8C%KtEYypPPrKMB?(1NaOZ%< znw(>EGk!JOfio`KFgaDX=;OdVIv|?GIm>@?i6vX`coV`H4#`lN!M&(Ht`e4uFD54^ zzC-I`Tb_-;FdY_K1N#&8aH3QS^rt$(>$5zfLBz-k^AXy)c`aSAQZYBd6IZIKfHP=w+9xOBLS(^lCY-!1Kf@c zgMUmd$y+WB-s0Wp)%5TQ?>$l~L10VEtd7mrsk*9l=?#9SrZSM&pq?VBxKV`<~}PK;2O;do>#lZ8}f< zdSj~_Z_g)3!U9O-stCx~_YDuoit;B5tt8+6UZE+4X}HENiMZs(0q;`|6-}Y^Nz*QT zV>by7ffFy%UfH_v^mK3gi!=kU7lO;doXmgc~xn~9_~ z&4nE9dV?)C*XaYTI4UDjhFhOW^7YoOgs#Ls_I0E#sD>A#;Fo8F9bAgiJActx)&AUU zQXk#2SB7I9tfxL5#*Eh|T}-<5hc13l526JNJ*OohG7#Tz&9Vd{Nkl1!;U zs0r0>7{ZVw&VOa#0t&}MaZP9`j#X8|HvRYLVXg{Ji#UH#`BLZ~O9K6!N3s2(9P{h% zMi_ru2K&V(@rpN>(T4{=fn$Xn-(8@ETpbt$hX`A$TvJ0ss*S+%@B?~J(g@loL}R@0 zT(W#iE_yXfgX@-P`gVI0J*wY}otu?tOY1NgUb~KcYg6!KYzMu+EFXMVtO4JSCt!3! z8}GeQ!M9xAcgs_6*gtg*mAACS$x4%HKt?5#e(wdlbiEOsnz9uqh-wfoTh4#rn+ow4 zavnQkv@ zG;@en#Mm)y@|zdJ8Wl%?w@3)v_eM43fXft(0Vf8b z;#Ot2vx3jLvMQ*Zx)sFAJOlHqUtq20H!6}d1!inQEbG2S{4UI;>uelwZTuey^_Im| z)u}L`JIEZ`($5O5QARn3Y3!;EG0?Rlg8C&Zz=F6!A~X<8Cou`j zp&43r2nwA$4hv#pFx1VN9Jn$QJSR2bi|*y1_~{xxS`<#6#O}g7e@r0TBoxkzSHgCs zIrNwJHM;P4Cxm>^B02Zs!8~Rj+-+Gx*L-)vaT6R!U2+MNp1G5fZFjM3e=RgG)Wuxe zzx0XNK?p69q^mkMf~5YEeG}M$S56Oy^1*m{x%g9hXCR zOdvO3(h1K>o8c9c!5&Q?dxTvkkV6 z3iC4G=YXc~UR0cygM+U!uv)3GI(MBWeNPTkE%pf+N`6ObQf=`37Xf^#RX_&UPKDo6 zMfhf7I+^{?R=(qtu%8#^Co;~ z>|lkQR8gK>2UBm(xp2E0&(`k6fkr{N*O@~E+PJe!=Uy^&MurVBJVy4dScHotL*Y!z zO|tFJ3Q(FJMSY$+BkOvBTE^7Vt>+hD*^oP3&G9#;yod(gQ(2HZ_Y79o=z)_Omyvp+ z3^cM71%h{*&EFCVN5lTXdxMXV_;&?tiu*ukTvkE@JVvFiy@ng(0M>D{@z!#USSgqa z`+ZljIcL1l^IJCU%=d&|oz*ziKNQrVit(#-r+S?taA1Kvc{Jg&$95X?Q<)673ok~1>TpbIB7P`(vd$~RKa31tqumIfbc(%z4WYVSR+h#4^yxEo@HS~~7S!=+r70JOy zlWAbtxC8Fr8poemGDes52=Ht?r@~v?uhgW)n3QeEp;z1@$$5(_taRx`T#~8Fvv6nN zhrS@ljC)6>H7M{ElXQ8P%JNCYv3OkC?uygi?S_ZZcOfI@JC)Lo$3m@4vhc_|R3A~H z+5szIp?()F)m%wc%o6dOgI)EL#v-!(>q*e5b2oEUdxLM}N@>W45|rQCKu$K?rsJEm zVLrDf92i$iW&fT)|8qe!yfPXjpOrKA&yv{rxoNoKa|%`IOd};inI!PiT;8|L>tN6+ zOJ*vF3(5*wiXC(DD2ljY){pS^dnUntWiqtU| z*F9b?-g|a+E(Z=8>~iyPJG^bbtl<9u3sC73 diff --git a/test/input_models/QONNX_QuantGemm_NoBias.onnx b/test/input_models/QONNX_QuantGemm_NoBias.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3c9d8576374732e84d727c738a6b9bc88aa05f78 GIT binary patch literal 17896 zcmeHPO^+ML5p7DlHa+^H2*h4XNkAw-4g^-FoJcVYbJJb~3E+cF1qcx2!lAUIT{!Z_r@rLiACP`A-#sNwq*@gG2->Yg;I&}TL=5MFt z*Td{^`sB02;b@kfP6o5=O=J3P%s89o_LKL6o|TJoP-vOgSMT+WhPi@*BQ zSA+8`X)ND-pH0U7i!o4=d&^H=4rl$h!;{%rbM?WumxIx4_#4GA#qnPHc)9ej4d^v+B5AI%N_UYqCdw;>7?R&}o;QZoj@C#$yAAPOX zzMJg5%x2f`q2+8c7)`%is?KkZ7t$vGt0f^xa(hX(a{m76U8v#eqd>#eCs*Y?V340z z!{&!CzJC7v@BJ6lT%<++`S^G^n704zYIR$;joR^bbA07o%^35OF@BD}5$+2_*O&s) zIx?n5f%O)mz`8}yW3iqi8d$f86zd&Ch-I+u5eDlW%0XDA z8LNy>s=bUa2VPt|?N8#9YB!X-MJvX#;F9=4tr$zA6=U_FXXp*{8Sr^Th!uOodo8-AgZ)7Soa9ppVsG;a-oC9XN^{j<>wFpq48Ox6=Tu$5?w)@KVH=!yAKL3g9jim?jhYJIjxyR(ma zrbhwYH9m76Ykbb(vDRnqV-NqXW)O8*)YU+QSPnT_pS3eVJ(HI3S@7kE5KE&K*RIj3 zm@mICxsPK$Nq3K$wrRz+)Be;xLvM-Cfv-S>Sg|+E7iz^=j`p}a`_waGpqGK(9-;9$ z_NJ;wE%m$7XYS)1^;oJ2PnpjJ)*XeyXG3|r-+d~rSa;XmoWU}mr5W_lT+%&7KWKd} zHS3@lAOytn3g}q69Vvj0kqKCZQ*|+SO*qf>zwbXxjEPNqvs(RF5 z-6N#F%ICm9&_h@SLhG|dKcq^Z?K}8fpdLrDJNwqW$>B|z&l;`RX9pau&t-gu{E5DE zQuKq?=N9@w>$3-!#3!9a+{aZ~F_wo8nwcOh?M!r_fp#W#XW#X|6%6`8>$9bPz+>Ud zHlGc0JVN7hfjq6x_8olI&IJ9wFXp?mkIsf1I%s{~oqfxm(ar>&+uE7P-(eO0}uMsJHp|e)fUGqJ`SY><$ zTpkf(mGKqGYoVqPE1&PqKJwW^2d&S$vv1im+L@pp%@LZJpdRg@M^l7_FFb>m{J~=* ztR8Y&hyv@50y9W_9udw2VM#MceDO>WRvBLkoOr&<_@vs)_@vtHQT6!|V$uF2z8w3D zb*KGFe3I^la(8E6Wnaa*J7^wy(}OoHgrs}yO;wL-XM%dPj4z%Es@=n5t)g*4A4d=5B8gcWujn5$Xb04eb89W_C%;&&YAVRG0ybAe3tr*LpCh@Gr9%7%hGht9q zj*$35t+;lLR*a?6YE`>Ns}?jZph4^n^QBm4t+;mDpGRnXE|3>`!+Z{$vF;XiOMH^< z7IkTTru|8LlJ16bcW2**HgHKH z6FKlSS{dwrQ9F7)p21kF9&+hy(9VQGt|JYk-&QtcY87%PPiTAwvqF_tF{5E`E~S}~S~4qBgEOcdn&nm5eM?M>b#3$))sV+QKS}~UC?#@2)S)&#E ztepw!QT0q9S346lUpYM1`pkXo;q|uK9dZhU)K?CiA(k|Q+*hm>W2M+1^|5B2X@3QA zMSN22WqgwE+L@r*HCkEdlp{1+UDb}gX?@mc#aQcFwLhxYa4A4|x5Za`*nYSD%f-UR zJr|2r(|)*EjqkizEPmNOSS<3b5sRxQXvKZe{?%e-^BRg3>WVYIl42b;yp3TUcH?G- zb=dfph9&Gj4ptX@H18#M>b2$v$<4aEk=&@;m%m9G@jVV3N#8u$xU@pb*}TW%ZgLxJ zbt0np=GSYNPHbQt?UtLsdfd$Y-9Hu-aM>m-}L znhY;y!||y3htc@(CGLf2Ew2=zYfwIGU1XEPSvH!EC#}W(5ueZn5}zzywpM79dQyAS zy!q+VXOs7n+CQ5&p8cL)j215%Nqr0qx;mw~xQJuY#8o82*;${cn)|N@=hJNRFli#G w|LSZo8fEAGrPsXp;o2=L&G$QxJHLDK>C?%<`b8$qIu@QheYRW)8{&H9|7ZZHqW}N^ literal 0 HcmV?d00001 diff --git a/test/input_models/QONNX_QuantMatMul.onnx b/test/input_models/QONNX_QuantMatMul.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8d4930c9084dbe9b796ea0e4d40be3fdcf1c6070 GIT binary patch literal 9907 zcmc(l!Ef94700PmXN}@KA zSy@Jko`^z6A9&a@3Iq)skH1XYn$89Xux}!SnOksy~Z&if0{9h*~JatIOi;5_c$)M>(loHCa66$$eOZ!lV8y zo?})Xo}ZAOOB1O-izn6`g?N5Mr})&L#p@DJkY?0D>QO$Ehc6<&gY=9YQhye2No=7w z%BWq>P(F(%uRJlH6wd~nyg!4-6ubnFq-Pu<^=I*vOEZ`u<#hPepT*NN>f`y5xO$wQ zt@^WgbK*_pp>n^}pT+Cq2_*9Air}agix=ZbnS;Y4pNCKVSv&_X#?vZ1>)h3!#S4hl z_e=d*yj{F=l-Eo?&B{pzuTQ*(Hjq3qwX1H6ml013S_shYucrCoD&5_H) zqkUF?7SF?%GOG+vxzFm);sto*>ne`)Ow^ynbHtly4@G!(kovQD<@aMre1TH=xGY|W zJYve5(O*pM7rs~9uXqU_SL8|_toAu1Zt;{$?@WqMo*7bq7EhiH;(BlT+_lf@&*Ew3 z5tQ7glcS3B2GOMe!i;N;)TR#NI5m9E>eFM zuaBp`U+T}|HOb?l>VBy|ix=PvVCssBx|iH7UdkLCG5I{|*+uHl;>CDc#5IF;)W@g( zEMEEjI3Yem*%|XjCU3jQflsr_pEb|M;`Q)sAg6rP9zIw-7mJtT2~l;w^8O5-W~Di3 zPUSwUKZ_?%hC1XqK?%O>Z1wKdc|GE%sJdV3&*JTor@mk6&*GKekEqBM;9DSzr`jB| z@`#uF98i!lRrPugp{3FFiwx=aHkn zU)AqbmA8(!Po5sqyHh?Ji#H>lQFDqyq*=*l@$}wk_PR8OCDIHmUd*h*!jp0D;5&h3 z@sw+UdU#Hi51*ck#oHmS+3yiA_gVd!JbVeh>VBy|ix=W+A3yGK3h&*DXRJd~0v z!=ru9`!jf&RX|+6uHs1jS-kT5(ZsWdB76>6ypVW(zjB|!(>u^XVrr&Y(VxZJ!IR+8 zOm!vHUHx927C9EgQ>0v)X^KxZsXvPs7ar|`=Gy13{w$uDf#PV6r-`oO&G0Cv=F~+2 zQcV`G{C-sTODv1$k*B_2>d)c@XRBF7McvwG^=I)MG5I{=yJ(KopT%nt(+t*8A5Rad zKa1z^oDiF#>}>UHQs!Xs@*Mc|F6de7imA!sZQyfwVseFemZLD8ZwBPLahs#Z%ue^=I*R$))!uBA2I})UKK=UPe6kVGRn8 z`m=bBS$TLa>=zu1CsuvGs^2Rzz2tS_B}g;spb)7plZP)NzJv6P9a4W5Z%J&SILfG9 z&rm*#C$Bs)o)pgpoF>{sb)ITZ@JNB;==6N`-qm@^6%wBz<wwES`fG<7pKh?X&u`cmc8cepSC$ zRo*V%Im&A$pJt^wSiC;*9@;?i#MG|3EnY@EMIq9x#8iJ4FTtZr9?fY7pZc?SVd2TR zckrB`**T1}%e)1i9`RGOj;DMVES`GU!&BWa^=I)CeD(cuuq<8+uWEDT^6+S%)t|-l z@TJTu!&C0F`m=Ze9{IY8BRv!KXYm~ICfY+0o*fjyvUv4>52W&OS-cK;%9&>L7gPI% z?^T7TnY$ua@?f>kA#sbRTzY3xeDchY`m=cQY!KIb)90>zR(}>xGmp@mcoTU@{h2&` zj(8UZ55Do=Bdiag~$SHD*=HKokT;gQcH*DjhP z)vW$3o@TI)`ow!kvsHh(>bw(TGn9#iLUa{R%q4NnUYC68&*C{eF|~(ymMB9Guf@~5 zpj->o!*hznvUti>-!JuN@jB!=K?(74pVgnmE59Etycr%>%-!PYy@~L6D5ZAwCRx1P zhczfXUFuPP7SF>|->+R*7O(t%RQF4DTf79GX4F9;TH;fG7B3>cgA#d=F7;;d7Q|DO zkxMg8@u@$HCr?~>Ht=Y!d)zHvLQD!2NAvSl%@!{tK10f>Idzfxvv_?x_5D(R7OzPj z4^{U|{aL&KUjS2ARMfrXZt+s);E2iRQO_%`^XQEJd7LE%X7=09d|wF|LxzK ze_z#g`4~%HoQD4YyUKaj&(jnmy_L_qJUz0SD9U ziqlE$xBVMyjrN=VYin`SzqaPS{;eO>53kfs;Wu|K9l5D{9G=UkaDL^tpsfjn!f(Rb zS(|#6@Uy1x1vfr>&%f&2QqY*Z>)*`JWeqr&H7E{awVTt)@o@5`|JK3q=+R^pO-G00 zwwvsazumeQt`0=$4A;kDG@NWbdb&Rz51(zVo;k`-QUppk5PJp$G(?tCC=8Mtv>EP?p{!#n-Z?_E5R)qbl|_T<*ZBTDTx{;#|DKe~7SKbdIa A!~g&Q literal 0 HcmV?d00001 diff --git a/test/input_models/QONNX_QuantMatMul_Add.onnx b/test/input_models/QONNX_QuantMatMul_Add.onnx new file mode 100644 index 0000000000000000000000000000000000000000..371eb4116d6d13cbc878f75769cf3b24ed72967d GIT binary patch literal 18256 zcmeHP&u<&Y6&}jJn2puY1d=Fr3oKl;2LZBaH#Pz{5++6ug^@UPfdEB&;L;K;(~(GJ zMZrN%bMK{>-uEx)wLlvM^MCZRe@eec@=3$t+vW2ihXP#S!P3t2_r3YP_lC3&)4bmP z?dkZ{aIkaw;Ip0KXfim=`;)=J&a2_~le2uVb2u0c@MfGp`S0Hv$v+#(wc+UGY?8b` z{iAof-#;28t=TU>4DxaBWDJz#gV~#x!%6S$@L=+~J%91tS${Mc{xCSWJqMV+xO?pd zz2DwUu8;b8K7PApipIwFC&^|$J{uhjM~A)R@xkEkwLg9R&GSDLjVqLSIy@W=4z?P| zeDd*k*fpj^ zq{kdc%B@?y)zLRu*{lIB-dPCD!BGV{as_7%QvktGMv4(!D^Q#OJ|f5%e%tN*-%{#@-}82fh*!VzrPf@kzQn)TH%UqZMO$)Fkm0;4{P- z=-z<_8G^m3tG$c8X?-@xmH0xf7|T*!c-Wh|+70$rAPj0=ReOm%iO+$LwPGv_ybLvK ze74Bb`dkv1z$eviC@=7)u6Dbtn+bcm>!Fqw9{b_y9C>J_teAZ~iSj9O6Kxll{XvJ8xKdsLituC;jf!1e@R*dCH zgC9{d>Rwg5Mk~hhz|;C%*H;!eAyyYUXni)w)%t9q13fWcCg^SyS}|6MT&>S`Z+-Sr z&vYrEyT)hkV~x)q9&3H(J}%(j#SEe@i@I8f5X&J)>$7$ysAsYYJ_|mN2(dI;aqSwd z%JbRhCHHa6C+S|Grd3*T?X*9&&(K@qbKolxAy(`S^MzV5mZLqc&p!1`80cl7cY)CO z9D7sOqn7$z=`;7SM?IEm!c*mQiFHSz@Yzt_`g@;BE7slZF3(_<&(aJQ&|K0zLqBMJ zuHut=w2DtUi|YEypjSgI?M!&+m11uapV*_yndl;KefG_ICibSTM=kXq9t&T{o4Ot~ zST7J#U-@%jAm|~i5~20kq8~D)&-NXBE>Vx8Sf70h-gtOZ<+DaB_SpeP>vI*KA%CLJ zoDBV-^*Kd9XniigCGkmT5%+POR*Y3Z2hB_nmUbpO&_FvA>$C6j-wFo(p!L~OKj5+O zWtGncIR!%FbBR2y&-NXB*3JZdzc0_%XCIvn9y(}!UY~ulp3%+(o!i=(@b54a>$C6j z^O??e>SN73ug|{O|C4C6;(sd`Iwzy=AJX>$IvXr{nEJT2ho2D{e4(>e(p~d8!B|y% z23!Rq#H!*ek(Z*T5X+yh&pz_GfDT%p*Jt0XXS6dxJ?asfnV=r+phq)=g)cmVmi)nE zBdjiRQbdV$M}Zk6z5)@>1Yt=tNPO{35LOjm2Ap`ls`#YZtN5ha?Oy%)5n|E)BtDP* z#k$k}BtA)ZL%Hj-uePsZ-5oR!z3IZ66d~yzdsEk=+L@ppt>TMkf@&||vDRl~F3lkE zbr5mw1$2=3Joqee;<3o_h;(o9bE(9-qmbsgg3ke`jId&Fm`~EZk`J|6aE4ggnJ}ow zBP71!90DLhENKQU<>85I@1iEH&#^ZSHI)cS_bR?j(A@x|Kxllf;&bQAK3ixm@r7D( z?Kbj8(u%Q4;u82I-3{ff&pvqu;g6)d+pX@aSaz%`atLan%VjaH1M(rRA2MynK>me3&fhWRqAvsPR??XN&+d@hj}dc%AUoU!f}bxVAb z?iO`veWv|Me3I^la@S|yk~fl87FyB6eXQ}huCHRXgIq@%NZ&Jo73*HbC+SZ6)A}5H<581ENV?Ph zBtA)Z+FwC9h*h;$@nyh>bvL`~vyXblLIwF8`atWmb|$EH z2OUa;W}dY(L0FkWD@%Dl>1Tq@h5|YmM4eV8ty3NztkH>pU@9{20{QqHWf81=~o`mockY`u1v)%1q zP8XIg1evay?K{)e_;Qfx;uqUn(?z~sV|qsit+>y(e>GiMzOQ4BI_HeD9!vHlN-}s;*LLHS$lAThqH@dblg5H@_gJeRYzb z4f4Mx?;i|K_w(V&WH=tRzs&oiZ+9N1vpZVoUYXC*lR>^S8H`THc{;s$Kw&PcJQ**LNprBdfRnx<48Xj(W3JX0z?1+ literal 0 HcmV?d00001 diff --git a/test/input_models/QONNX_QuantMatMul_Padded.onnx b/test/input_models/QONNX_QuantMatMul_Padded.onnx new file mode 100644 index 0000000000000000000000000000000000000000..a4d0aadc15469b8fa4a61fe7c51e2f86782110f4 GIT binary patch literal 22609 zcmeI4v5zC?dB$f?lzgJ7c(g@)mNvjZ0RrNHlPQHV1jD!ZBBUUMNTz}WAc6&!C*4X~ z`?NdW+k*^O1A`@tY0|Nnl1OWc=w0jo?M(Cr{`qp|8Vy=Uz}bZe|P%y z@|*qoFMj>v$=T)UZ%>}ST_>pi;@ww2*WW*Uv;W%JlZ%VDkxE-=04`dH2=7|HVIk_P52}D{?+R{rc?W>4QC-Kl{)L$6uxWLH(b*s^6;r)G9l!|LXg?o4@(K@FvM0x$?hjX84la zzg5fro8LaU{PxAO_6NKFr+$NP)qiTw zqW-JaEb6~`NqM_T^2eRU{)5kd@!4my%BeYxQ8f$0o8tK$aWU6d)PwXmUJp-%evcEm79TYep{psqknzUYwmi`1XvWq1a}HG=~*#i#xp?+9GHNgI>D)P9z^5n2 zBWVWh`>M^;nNh9;&jJmR`g1(R^p2+dHv3$i;j7!c6wjJGJrp7Bvuby|Kzxj1sm`UE z9Iv@Q3UUQJDN=uqr#S@r%kT_pk8(O*lc&93dWVh|k)yj`?fa_DJHR_7&jji0H1Edo z7Q{MCrEBPExXF;>qqd9DmX5e@kvr1}D$?pi?5|-mB*9J}StdtL*-izbSh->z9 z;>|v*Kg+|H<7@Ai`g6Pl-vDVAf!ZVLQGbq?;)zf}t`d*-x$4jGG^?1nd}GCt`g6SI z{^;SEqZD62j+YSc?pNhAJe`3N5>qoR>;4>Xh9}3Pnd-@@yM14+>Kq&51yU}}w7{pD z)Su&JwMV<4xlZ|2e~u?+tT7l!L3p~oHIgL?_RFmU1_eXob#B#idJl*|Le~uSl z?PitMb!(s1pW_8$@E>f#~TpS3=Yr~&jhJI#|wCt#1^Q$+P#~CIXGUG1E0=< z-nE{LnjCM4FW||@mEhT;_I=gnCHQm}#N43C;jTZ&lSjGQ`=$OIuf#VZ*AnG;w9f@{ zycM4AeyKmlJ0h3PO-imvIjLPWIbKP;@?ni@kNR`Gz^o!XH}(sTE@Rc%)cy zw7TA%yDm?;65rT!dmO^zOlP_xhK&+*21VyVuR!ci@bm*FXx zL%<_ngirlBUVxY38Pp!_v-)$qm{@ne+V@qPcZ7G1s+!5CS!oWAHzhtnLnKc|?W)`H zO5z1dkY**O`g6P-j~;n6rx`x==Xgo&Dfu1YS)#=?jH}zc4W0?{6*|DvoCU{I4|6>2 z{ZfC9m*ea1SAgYs1H7s&kSoHYeO7;t7vU?IRf(tBXZ7cJF&_EGiX*)f_2+njcn{4{ zif4vWSdQ2I_dp>ZkK>KV)6BG_zl_>%+*d80W*+KX`QzO_C&V33xpZaO<|$ee@1Y2(Kg+`xh>uY$;ZuK(rFgS>9ot1D|G9y=&f$<4y1kQBXc=Paf~yi{lk|64c(W zsz1ZitTYGBso7`s=XmmzXhfbR%JG#~yR+BjO^B~hd%x75B-3@X$I~4s?F1xDexqC7HEjnpW`XE#XIG<+2^}> zDV{ZXdMHBLXVva_f%q835JZ$@0RpA&EPS^Zfaz8qhBzto@OCHMwNvk254QIGm_ycAD_ z3UZZrw9i$4hNoG@#N`_+j?|yyHTOpk&m5)r0&={Bcz3@lpW*2YjF6a`X<7H@cr!dX z9?euwPTlSMYE|dh5HFB&X{H4})ujF$FRMM;1?U7ParIHZQ@avmoXMO%8Yc zIi5Vq)!r}l=XfQ)5xJHq$D@5NkmIfJboWdBIo=VubZ$~|MaoI-s>$(6;*}3;Tzk}? z;{|3F;kmJ2a2!vp?tZoJD>A+0jp5};Ga8`;sV>XImlB^Lz2kt?pW|(bZ4^f(wd) zP=uO&R)3B+#uH0*t`v@Hal8yq!5jh}`67Jk&+!7h49}qUXrI-e}M4o1*CH-a8e&fDs@ig;L z=gJ@N_BkQ$c*>g&PPcu)^ns^UINc~wJzCe78VhNx6 zb3DzoyI<Ke-?sz&kDV_)w)UMtn$E$o;(E=Kg5!m+E%B9G+%0LJ8X9Q-6+^5}%=59;8RTIo^hNfl6{|rUgFr=XmmD zwP%P&bDi_+csVgCRvfLbcQrd+LVSUgQ*#<4_2+m~Jl*|Le~#B9PlVe0rT!c*#uvlX zlh$=_`E|U4IRs+zMbvYI)Su&Jcm~8Zg99|hr~VwTxj&Y~7pS~q-pKL}>m2wrtLk0z zZX9odXNZFGQG4=u_g);Yz>}c%epUS$o@S*vXim*Ot3St+r$i(2EK!cHyxN_;E^k77 zh1&b2{v2;kp6-6BKgVnCkF?Ge(BAzQLgrWsXxan@r}r}L^&SqbAcRhg{Qk;>d*0x z$fa|Wk}Fb9YFAB;R}!y$SmWBG{v0nbs|e4H{et6oVs-bceP5C3C2tHbN1D+HB}jEy z9=??L4Cx&Qr2ZUlOKhV!Dyd!XP(H_#SDp+{foBM(hvuluQ|&n(DOMbqlY5Y?6dlFyfL0ws&l1qREy(fcnana@W>b8Q-6*Z z;AMCQwMYA`{v0nR*4?l6ebwe2;a#JuX7XuPnuFs_iBHfF$&*pL>UO-6c!3h6S&6Cs z952VCM;^^-hEM%DUQ&BXen)tgXmJhW>NanKXF`014)8Q*!SU3?98Y_{)Su(!_`3TQ zU^(6ZuWAeAituQk)t}=<_zGrK;%W9-{W)HYN4~M*Nbf}bIbI;%LvxhknV}Sx<8}W% zP{_yQcq8&OGcD;aqxKv3Rg0&YhdNjOc(>09amQ0GotXlkJPV}$98aDhah;nfzuIT@ z=XjcViq^z?C_?Jb^6&-XV-!pH)Su&NrrrHgkB-;eAAwvMp26YG`>M^;+|xQwv(N4O zDx;=?Sp_`uMdUg{YowahpW|r;2WU!sf;3z8r>DzX5?i2BER>+Tcw%meYxa8NQ-6*Z z@MP4U;Mt-Q1-y=@v!GlXG{Lh%VmY32b@xmCIo^moOOz9D_F4ToUUPp8@RoQ&J$J{` zxk>RvsGxTBCOKZ^!y4C~G4-fF$BXcE_v;9j<2Cn3d%sk-do;s#0ylCOEWF-sXxb)C#yX}JeupAU&qUdNwMN+eZ8yM@e<+-q@0@57^y$U zo8sy2m-=(O9(f|v-Y@m%crm^hrk=E}d&{rm70e+JlP{v4Bc%QuFT*n+t{EJlDL(b* zc+LH>B)&l974t@xcUb4Zr&(3+ns?)P6Ffr{l#kkz$Gi99cm~(n);w#kNFZJhmbMkceOZ_=sbAP0Dt{C42Ii6|@%qk+@ z>~l;_j@R5D?fp`Jj;9%n$&(|!lMxyq$Lrylqm=jzRWqP2$6FEK65A+_N@`bsj+fM4 zd76C=hqwRzAznawa&k$ULHoXH^K@nkJPDo!8Y1=Qc#3WDPWf&2`7U0HXHA|SijekM zwL4xQK1Q*GPmgMHyfyI*732zdQl$PIPjd+Lm*E-I9_4hrCQp06^bQ>_B1d<>+V@qP zcYt?Fo(a;~Y2JQlHU=&B`n8Nt__;t zSt%bry%)!u5!dYJ#G8Fqf0l-LJ}Lcsc_kB&KFs*8Mr&3{Q?nGu4w*cl*9t z)j2lA3#43{X@O5QsXxcdYL9k7bDi?5{v1!tSaGz*(?fUh7I>6Xa~h)()N2KgSEi8$1)@ zD|CRTISY=b9_D!3`=$OIFUQy2uK>&O26$ClAXkJ(`>g&PFTz(as}fJM&+5d)~4@gAC^6weH$upF=Z?}0)-9>*Jzrmf0lvQ+Mo%Z6%xzwl&ia6>d)~; z5rLcDiy!yjs=knW%olQx>(Aou z_bxA9oLqd_f8*)N^Di$>)63KIv;BWatB;aC`ZPH^KYR3?kD^|jBo|N4e)Z^sWcRgN zePQ>bBt5x!^yN2C&dyGrJ=(pOr24e!Pj=rn{YmwQq()ub@9%xP|H_B&zkl(A{@$^Zi%9dh+b~$;I3KeQd`y z^mqrYoA~n^-}2r6!RWowU%&sshZpbM{9Nw-8?CZ09^CwX?EZcJulGOv=?5SFFIVTJ ADgXcg literal 0 HcmV?d00001 diff --git a/test/input_models/references/QONNX_QuantGemm.ref.hxx b/test/input_models/references/QONNX_QuantGemm.ref.hxx deleted file mode 100644 index a804db0..0000000 --- a/test/input_models/references/QONNX_QuantGemm.ref.hxx +++ /dev/null @@ -1,1032 +0,0 @@ -#include -#include - -namespace QONNX_QuantGemm_ExpectedOutput { -std::int8_t output[] = { - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, - -4, 3, 4, -12, -6, 0, -10, -1, -5, -7, -8, -4, -17, -6, 3, -1, - 10, -16, 21, -5, -6, 13, 3, -7, -1, -4, 3, 3, -1, -4, -8, 9, - 3, -3, 2, 2, -6, -1, 0, 5, 18, -14, 0, 5, 3, -8, 8, -2, - -3, -10, -8, -3, -8, -2, 8, 6, -2, -2, -8, -3, -11, -8, -10, 9, - 2, 5, -8, -3, 5, 0, 10, 4, 10, -10, 1, -2, -2, -8, 0, 9, - -4, -13, -5, -8, 11, -7, -8, -5, -1, 0, -6, 5, 6, 8, -5, -13, - -3, 2, -4, -1, -9, -8, 4, 0, -2, 3, 7, 4, 1, -4, -2, 9, - 0, 8, 0, -4, 9, -11, -8, -6, -14, -11, 12, 10, 5, -6, -6, -2, - 1, -7, 2, 0, -1, 3, -8, 0, 2, -14, -7, 2, -7, -17, 4, 8, - 13, -4, 7, -4, -9, 3, -9, -7, 4, -8, -10, 1, -2, 6, 8, 10, - 7, -6, -6, -1, 6, -3, -4, 3, -5, 4, -10, -6, 1, -8, 5, -4, - 2, 3, -7, -7, -4, -1, -7, -7, -1, -11, -4, 3, -8, 2, -5, 6, - 16, 9, -5, 5, -6, -7, -3, -1, -3, -16, -2, -6, 15, -12, -14, -11, - 7, -1, 9, -8, -1, 3, -6, 2, 1, -5, -5, 15, 0, -8, 4, 2, - -7, 7, -1, 10, -3, -4, 3, 5, -3, 0, 7, 2, -4, -7, -5, -7, - 0, -5, -1, 3, 3, 3, 4, 7, -6, 5, 3, -2, 5, 12, 3, -4, - 8, 2, -7, 9, -1, 4, 4, 6, 15, -2, 7, 3, 5, -13, 5, 9, - 0, 6, -6, -7, 3, 4, -4, 13, 9, 12, -1, -2, -9, 2, -4, -4, - -4, 7, -7, -2, 1, -5, 0, 2, -8, 10, 7, -6, 2, 2, -15, 2, - 12, 6, 12, -5, -4, 2, 0, 0, 7, -1, -5, 4, -6, -1, -7, -3, - -12, 5, 1, 11, -9, -5, 3, -1, -10, 0, -4, -6, 12, -4, 0, -7, - 2, 5, 19, -3, 4, 4, -1, 1, -3, 0, 9, 1, 7, -8, 0, 19, - 2, -9, -7, -4, 8, -3, 1, 1, -2, -6, -4, 4, -10, 9, -11, 0, - -4, -4, 7, -6, 4, 4, -2, -1, 9, 3, -13, 5, 3, -1, 6, -2, - 1, -4, -6, -3, 7, 3, 0, 0, -2, 10, 5, -4, 2, -2, 5, 10, - -8, 14, -18, -1, 0, -6, -6, 15, -10, 6, -1, 3, -7, 2, -4, -8, - 2, -1, 2, -6, -6, -2, 5, 7, -3, 7, 4, -6, -11, 9, -8, 4, - -14, 7, 6, 8, 6, 1, -4, 3, 2, -1, 10, -11, 10, 7, 15, -3, - -8, -11, 8, 2, -12, -2, 4, -2, -1, 11, 3, 0, 1, 0, 4, 1, - -7, 9, -2, -4, -4, -3, 0, 14, 0, -4, 5, -3, 3, -3, -1, -5, - 7, 2, 4, -7, 1, 6, -8, -4, 1, -3, -5, 5, -5, 7, 5, 1, - 0, -5, 4, -2, -4, 3, -5, 3, 11, 8, -4, -1, 2, -8, 7, 11, - -6, 2, 9, -1, 3, -7, 14, 0, -4, 4, 4, 1, 1, 11, -10, -9, - -10, 4, -8, 6, 11, -4, 6, -10, -3, 0, 4, -3, 5, 1, 9, -5, - -13, -2, 7, -3, -2, 3, 1, -11, 5, 1, -3, 0, -1, 1, 10, -6, - 1, -8, 7, 7, 2, -1, -2, 6, -9, 2, 4, -2, 4, -3, 15, -10, - 4, -10, 5, -9, 7, 0, -13, -3, 7, 10, 10, 10, -13, 11, 6, 8, - -8, 3, -11, 8, -2, 7, 12, -13, -9, 5, -2, -10, -15, 1, 1, -3, - 8, 4, 2, -2, 2, 9, -5, 1, 6, 6, -5, -6, 7, -11, 9, 6, - 8, -5, 4, 6, 2, 3, 0, -8, 1, -7, 5, 1, -5, -2, -5, 4, - 3, 11, 8, -3, 2, 0, -9, -11, 5, 9, -13, -2, -6, 12, -11, -13, - 2, -9, 9, -2, 2, 1, 8, -14, 0, -4, 11, 0, 5, -7, -7, 3, - -12, 10, 0, 2, -9, 8, -2, 4, 5, -9, -5, 10, -2, -15, 7, 3, - 1, -12, 1, 1, 0, -5, 0, -4, 1, 0, -3, 8, 5, -1, -8, -3, - -10, 4, -2, -7, 10, 11, -7, 11, 13, -7, 7, -2, -9, -3, -3, 9, - -2, -3, -11, 3, 0, -4, 2, -11, 2, 3, -4, -12, -4, 1, 1, -7, - -8, -2, 3, 2, 1, -3, 1, -11, 4, 4, 4, -13, 10, 6, 3, -6, - 2, -4, -7, 0, -9, 0, 1, 2, -2, -2, 0, -3, -9, -3, -3, -1, - -3, -5, 5, -4, -3, 6, -2, -3, -3, -8, -11, 2, -11, 0, 8, 1, - 14, -10, 10, -6, -6, 6, -1, -3, 2, -6, 9, 0, 3, -1, -5, 6, - 9, -9, -2, 0, 5, 0, -5, 2, 7, -7, -3, 1, 0, -2, -7, -2, - -3, 0, -11, -11, -3, -2, 5, -1, -6, -3, -5, -2, -9, -5, -11, 6, - 14, 7, -7, 3, -3, -3, 8, 5, 9, -10, -4, 5, 14, -6, -4, 3, - -3, -12, 12, -4, -2, 0, -8, 7, -5, -9, -1, 7, 1, 8, -2, -12, - -7, -2, -1, 6, -8, -8, 4, 0, -12, -2, 0, 6, 4, -2, 1, 1, - 1, 4, -2, -6, 0, -11, -4, 1, -15, 1, 10, -3, 7, 6, -14, -7, - 4, -4, -5, 7, -3, 6, -3, 0, -4, -13, 10, 3, -1, -10, 4, 2, - 7, -4, 2, 1, -6, 1, -6, -3, 13, -3, -12, 5, -2, -1, -1, 4, - 4, 2, 3, -2, 5, 1, -12, -4, -13, 6, -2, -4, -3, -8, 3, 5, - 7, 12, -14, -2, 3, 0, -1, -6, -5, 3, 2, 3, -14, 4, -4, 0, - 11, 12, -2, 14, -3, -5, -3, -8, -14, -12, -6, -4, 14, -10, -9, -4, - 18, 1, 6, -5, -2, 3, -7, 1, 8, 3, 3, 14, 2, 1, 0, 17, - 2, 6, 0, 6, -4, -3, 3, 3, -2, -8, 8, -8, -7, -5, 1, -6, - 5, -1, 0, -3, 4, 5, 6, 12, -3, -1, 1, -1, 6, 12, 10, -15, - 7, -1, -11, 12, -2, 2, 10, 4, 9, 2, 9, 10, 15, -1, 2, 12, - -7, 9, -8, -2, -1, -10, -2, 21, 3, 9, -5, -2, -7, 6, 0, -11, - -1, 6, 0, -5, -2, -11, 7, -2, -1, 10, 14, -7, 7, 13, -19, 6, - 4, 6, 8, -7, 4, 2, 0, 1, 6, 9, 2, 5, 2, 4, 5, -2, - -15, 0, 4, 4, -15, -1, 5, 1, -8, 3, -4, -4, 4, 8, 5, -6, - 5, 6, 8, 3, 6, 6, 3, 10, 8, -8, 6, -1, 0, -16, -4, 14, - 2, -2, -3, -8, 10, 10, -2, 5, 0, -5, -2, 10, 1, 11, -5, -3, - -2, 0, 3, 1, -1, 4, -3, 1, 5, 10, -17, 5, 3, -3, 5, -2, - -7, -4, -5, -2, 7, 0, 2, 7, -12, 8, 12, -4, -2, 6, 4, 9, - -8, 20, -19, 3, 11, -2, 5, 8, -12, 4, -3, -7, -5, 0, 1, -7, - 0, -5, 5, -4, -2, -1, 8, 5, -1, -3, 1, -5, -7, 12, 0, -2, - -3, 14, 6, 10, 0, 2, 0, -2, 3, 1, 7, -14, 8, 7, 13, -2, - 1, -11, 3, -12, 7, -1, 1, -9, -4, 9, 2, 1, -7, 10, 7, -2, - -6, 11, 5, 7, -4, -10, 3, 11, 6, 3, 0, -3, 3, 6, 2, -4, - 7, -3, 5, 6, 9, 7, -9, -2, 1, -1, 3, -4, -5, 4, 4, 6, - -5, 1, 6, -2, 1, 4, 2, -1, 12, 4, -1, -7, -6, -1, 7, 8, - -1, 8, 0, 4, -3, -7, 9, -8, -1, 6, 2, -2, 6, 14, -13, -14, - -13, -4, -2, 8, 4, -5, 9, -4, -4, -1, 5, 2, 0, -1, 4, 5, - -11, 4, 0, 1, 2, 14, 0, -11, 3, -9, -6, 13, -2, 9, 9, -8, - -3, -10, 5, 11, 2, -3, -9, 0, 0, -4, 4, 2, 5, -5, 6, -3, - 3, -12, 2, 1, 4, 4, -8, -1, 5, 6, 2, 13, -19, 9, 5, 7, - -7, -6, -14, 10, 2, 3, 10, -11, -1, 11, 5, -9, -11, 7, 7, -9, - 2, 3, 3, -3, 1, -5, -7, -1, -2, 7, -12, -4, 7, 1, 6, 0, - 1, 4, -3, 5, 0, -3, 11, 1, 4, -5, 1, -1, -2, -2, -5, 0, - -6, 10, 0, -10, -5, -5, -12, 0, -3, 1, -12, -3, -3, 1, 1, -13, - 5, -13, 26, -8, 2, 7, 8, -11, -4, -8, 2, 5, 4, 0, -10, 10, - 1, 3, 3, 4, -2, 2, 1, 0, 15, -8, 3, 12, 8, -8, 7, -1, - -5, -9, -2, -4, -2, 3, 2, 2, 10, 4, -6, 0, 4, -3, -11, 3, - -2, 4, -4, -4, 9, 11, 1, 12, 11, 1, -1, 1, -5, -6, -1, 6, - -4, -12, -10, -1, -1, -4, -3, -7, 0, -1, -6, 0, -2, 5, -6, -1, - 0, -3, -2, -1, -6, 3, 12, -1, -3, 3, 1, 3, 2, -1, 5, -9, - -5, 0, -3, 2, 1, -6, 1, -10, -13, -6, 2, 4, 2, -13, -9, 1, - -6, -6, 1, -4, -1, 12, -10, 5, -4, -8, -13, 3, -2, -2, 7, 7, - 10, -5, 10, 4, -6, 4, -9, -7, -1, -3, -5, -2, 8, 1, 3, 13, - 9, -10, -11, 6, 8, -4, -9, 3, 5, 1, -10, -6, -4, 0, -1, -3, - 3, -4, -13, -13, -8, 2, 4, 0, -3, -5, -5, 4, -4, 1, -13, 5, - 15, 13, -4, 5, 6, -5, 1, -3, -6, -3, -4, -1, 14, -3, -12, -7, - -1, -6, 9, -7, -7, 2, -10, 4, -4, 7, -7, 9, 0, 5, 0, -7, - -9, 5, 0, 8, -9, -3, 9, 1, -3, 2, 5, 4, -2, -7, -11, 8, - 4, 1, -8, 5, 0, -10, 1, 2, -6, 3, 4, -10, 3, 4, -6, -5, - 3, -3, -9, 3, 4, 5, 3, 1, 5, -9, 8, 5, 2, -6, 1, 6, - 9, 0, -1, -1, -1, 2, -6, 2, 12, 8, -6, -6, -4, 1, 10, -8, - 2, -1, -4, 8, -1, -1, -1, -4, -14, 6, 5, -10, 9, -2, -6, 6, - 8, 9, -8, 2, 2, -3, -2, -10, 4, 4, -1, 3, -16, -3, -7, 5, - 5, 10, -5, 6, -6, -7, -1, -2, -11, -12, -9, -2, 13, -7, 0, -11, - 14, 13, 3, -8, -5, 1, 2, -1, -4, -1, 2, 7, 12, -7, 6, 20, - -7, -4, -7, 2, -2, -2, 3, 9, 10, -11, 5, -2, -5, 1, -6, -6, - -4, 1, 1, -1, 4, 5, -6, 7, 2, 1, -1, 6, 3, 7, 10, -5, - 6, -1, -5, 5, -1, 2, 10, 1, 6, 11, 13, 0, 13, -5, 6, 5, - -7, 8, -13, 1, -2, -8, -4, 23, -3, 5, -3, 2, -4, 5, -1, -12, - 9, 1, -1, -5, -2, -12, 1, 1, -9, 9, 13, -4, 2, 6, -17, 4, - -12, 8, 10, 2, 8, 8, 1, 11, 5, 7, 13, 0, 14, 4, 8, -2, - -12, -7, 13, 1, -12, 0, 12, -4, -8, 13, -4, -7, 3, 10, 11, -1, - 0, 5, 2, 3, 1, 0, 4, 10, 8, -9, 0, -3, 8, -4, -5, 3, - 5, 2, -3, -8, 0, 4, -6, 3, -2, -4, -7, 10, -6, 6, -3, -1, - -4, -1, 5, -2, -3, 6, -6, 3, 11, 11, -9, -5, 0, -2, 3, 3, - -1, -1, 3, -4, -1, -14, 7, 6, -6, 6, 5, -6, 6, 7, -1, 4, - -10, 8, -20, 8, 8, -1, 0, -4, 0, -4, 6, -2, 7, -6, 5, -12, - 0, 0, 16, -7, 1, 0, 9, -4, -3, 3, -3, 0, -1, 7, 10, -1, - 2, -2, 9, 4, 5, 5, 6, 1, -8, -1, 4, -6, 8, -2, 13, -3, - -7, -7, 11, -10, 4, 2, -8, -10, 1, 16, 7, 1, -15, 15, 9, 0, - -7, 1, -9, 8, 1, 0, 8, 0, -3, 2, 1, -6, -3, 0, 0, -7, - 8, -5, 5, 0, 10, 7, -1, -4, 6, 5, -2, -2, 3, -9, 6, 12, - 3, 4, 2, 1, -4, 5, -3, -8, 6, -7, 9, 4, -9, 0, 4, 15, - 0, 2, 14, 3, 3, -9, -6, -6, 1, 4, -12, -3, -2, 11, -15, -21, - -6, -6, 11, 3, -5, -5, 8, -12, -5, -1, 9, 3, 8, -4, 0, 6, - -5, 5, -8, 11, 3, 14, 0, 5, 0, -8, -7, 13, 0, -7, 14, -2, - 4, -10, -1, 6, -4, -8, 0, -2, -5, -4, -7, 4, 6, 2, 1, -3, - -3, -12, -9, -8, 14, 8, -3, 9, 8, 0, 6, 2, -19, -3, -2, 7, - 2, -1, -9, 8, 0, -2, -2, -11, -3, 3, -1, -10, -6, 3, 4, -6, - -1, 0, 6, -6, -2, -6, -2, -6, 3, 10, -3, -11, 8, 0, 4, -5, - -1, 2, -7, 2, -6, -1, 10, -6, -1, -7, -2, -1, -9, 0, -7, 2, -}; -constexpr std::size_t output_size = 16384; -} diff --git a/test/input_models/references/QONNX_QuantGemm_input.ref.hxx b/test/input_models/references/QONNX_QuantGemm_input.ref.hxx deleted file mode 100644 index 8e37bfa..0000000 --- a/test/input_models/references/QONNX_QuantGemm_input.ref.hxx +++ /dev/null @@ -1,1035 +0,0 @@ -#include -#include - -namespace QONNX_QuantGemm_Input { -// Deterministic signed-int8 carrier input for QONNX quantized GEMM. -// QONNX source graph encodes the corresponding real-domain values as -// x_f = (x_q - z_x) * s_x with s_x = 1/32 and z_x = 0. -std::int8_t input[] = { - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, - 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, - -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, - -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, - -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, - 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, - 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, - -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, - -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, 6, - -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, 1, - 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, -4, - 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, 8, - -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, 3, - 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, -2, - 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -7, - -2, 3, 8, -4, 1, 6, -6, -1, 4, -8, -3, 2, 7, -5, 0, 5, -}; -constexpr std::size_t input_size = 16384; -}