From 8a2d127d1e484c48c84ce805c3b3bab4be2f6da6 Mon Sep 17 00:00:00 2001 From: yyzhang Date: Fri, 19 Dec 2025 16:56:29 +0800 Subject: [PATCH] [init]init branch_linger3, ver:3.0.2 --- .dockerignore | 1 - .gitignore | 20 +- .vscode/settings.json | 5 + LICENSE | 4 +- MANIFEST.in | 3 +- README.md | 80 +- README_en.md | 105 - doc/image/bmm_int.png | Bin 21929 -> 0 bytes doc/image/contact_me_qr.png | Bin 67461 -> 0 bytes doc/image/conv_fused_bn.png | Bin 22646 -> 0 bytes doc/image/gru_int.png | Bin 49962 -> 0 bytes doc/image/gru_int_with_batch_length.png | Bin 54720 -> 0 bytes .../gru_int_with_batch_length_with_state.png | Bin 57456 -> 0 bytes doc/image/lnn_flow_path.png | Bin 44809 -> 0 bytes doc/image/lstm_int.png | Bin 51515 -> 0 bytes doc/image/lstm_int_with_batch_length.png | Bin 55473 -> 0 bytes .../lstm_int_with_batch_length_and_state.png | Bin 119690 -> 0 bytes doc/image/trace_layer_normailize_init.png | Bin 21408 -> 0 bytes .../{from_mode_to_onnx.md => export_onnx.md} | 10 +- doc/tutorial/get_started_for_two_stage.md | 50 - doc/tutorial/install.md | 10 +- doc/tutorial/install_bugs.md | 61 - doc/tutorial/install_errors.md | 15 + doc/tutorial/quant_advanced_guide.md | 58 + doc/tutorial/quant_faq.md | 62 +- doc/tutorial/quant_quick_strat.md | 69 + doc/tutorial/release.md | 8 +- doc/tutorial/support_quant_ops.md | 1154 +----- doc/tutorial/two_stage_quant_aware_train.md | 150 - ...analyse_tool_and_onnx_export_debug_tool.md | 98 - install.sh | 6 +- linger/__init__.py | 30 +- linger/__version.py | 2 +- linger/checker/__init__.py | 1 + linger/checker/float_ops_mapper.py | 476 +++ linger/checker/iq_ops_mapper.py | 451 +++ linger/checker/onnxrunner.py | 574 +++ linger/checker/utils.py | 254 ++ linger/config.py | 407 +- linger/constrain/SparifyFFN.py | 151 + linger/constrain/__init__.py | 12 + linger/constrain/cbatchnorm2d.py | 63 + linger/constrain/cconv1d.py | 31 + linger/constrain/cconv2d.py | 31 + linger/constrain/cconvbn1d.py | 134 + linger/constrain/cconvbn2d.py | 138 + linger/constrain/cconvtranspose1d.py | 48 + linger/constrain/cconvtranspose2d.py | 49 + linger/constrain/cembedding.py | 45 + linger/constrain/clayernorm.py | 33 + linger/constrain/clinear.py | 30 + linger/constrain/cmodule.py | 174 + linger/constrain/cutils.py | 17 + linger/conv_bn_fuser.py | 444 --- linger/dumper.py | 163 - linger/initialize.py | 822 ++-- linger/kernel/cpu/arcs_qsigmoid_kernel.cpp | 77 + linger/kernel/cpu/arcs_qsoftmax_kernel.cpp | 192 + linger/kernel/cpu/arcs_qtanh_kernel.cpp | 85 + linger/kernel/cpu/extension.cpp | 215 ++ linger/kernel/cpu/qlayernorm_kernel.cpp | 281 ++ linger/kernel/cpu/util_kernel.cpp | 26 + linger/kernel/cpu/venusa_qsigmoid_kernel.cpp | 82 + linger/kernel/cpu/venusa_qsoftmax_kernel.cpp | 204 + linger/kernel/cpu/venusa_qtanh_kernel.cpp | 86 + linger/kernel/gpu/arcs_qsigmoid_kernel.cu | 109 + linger/kernel/gpu/arcs_qsoftmax_kernel.cu | 199 + linger/kernel/gpu/arcs_qtanh_kernel.cu | 110 + linger/kernel/gpu/fake_quant_kernel.cu | 253 ++ linger/kernel/gpu/qlayernorm_kernel.cu | 326 ++ linger/kernel/gpu/util_kernel.cu | 34 + linger/kernel/gpu/venusa_qsigmoid_kernel.cu | 114 + linger/kernel/gpu/venusa_qsoftmax_kernel.cu | 210 + linger/kernel/gpu/venusa_qtanh_kernel.cu | 109 + linger/layer_normalizer.py | 208 - linger/layer_tracer.py | 407 +- .../lingerext.cpython-36m-x86_64-linux-gnu.so | Bin 21157808 -> 0 bytes .../lingerext.cpython-37m-x86_64-linux-gnu.so | Bin 21157672 -> 0 bytes .../lingerext.cpython-38-x86_64-linux-gnu.so | Bin 21097872 -> 0 bytes linger/modules/__init__.py | 15 - linger/modules/modules_configs.py | 24 - linger/modules/normalize_batchnorm2d.py | 55 - linger/modules/normalize_conv1d.py | 50 - linger/modules/normalize_conv2d.py | 121 - linger/modules/normalize_convTranspose2d.py | 82 - linger/modules/normalize_convTransposebn2d.py | 95 - linger/modules/normalize_convbn1d.py | 103 - linger/modules/normalize_convbn2d.py | 105 - linger/modules/normalize_embedding.py | 44 - linger/modules/normalize_fastGRU.py | 375 -- linger/modules/normalize_fastLSTM.py | 405 -- linger/modules/normalize_layernorm.py | 63 - linger/modules/normalize_linear.py | 42 - linger/onnx/__init__.py | 3 +- linger/onnx/export.py | 540 ++- linger/onnx/infer_type.py | 3399 +---------------- linger/onnx/scope.py | 238 -- linger/onnx/update_dequant.py | 6 +- linger/ops/__init__.py | 21 - linger/ops/avgpool2d_int.py | 229 -- linger/ops/batchnorm_int.py | 459 --- linger/ops/bmm_int.py | 281 -- linger/ops/conv1d_int.py | 332 -- linger/ops/conv_int.py | 412 -- linger/ops/convtranspose_int.py | 353 -- linger/ops/embedding_int.py | 244 -- linger/ops/gru_int.py | 1222 ------ linger/ops/iqtensor.py | 1387 ------- linger/ops/layernorm_int.py | 495 --- linger/ops/linear_int.py | 328 -- linger/ops/linger_functional.py | 1567 -------- linger/ops/lstm_int.py | 1239 ------ linger/ops/module_self.py | 27 - linger/ops/ops.py | 177 - linger/ops/ops_configs.py | 26 - linger/ops/ops_names.py | 23 - linger/ops/relu6_int.py | 157 - linger/ops/requant.py | 40 - linger/ops/scaledround_int.py | 120 - linger/ops/shuffle_channel.py | 90 - linger/quant.py | 144 - linger/quant/__init__.py | 2 + linger/quant/calibrate_funs.py | 88 + linger/quant/ops/__init__.py | 3 + linger/quant/ops/qconfig.py | 116 + linger/quant/ops/qmodule/__init__.py | 21 + linger/quant/ops/qmodule/qSparifyFFN.py | 144 + linger/quant/ops/qmodule/qavgpool1d.py | 48 + linger/quant/ops/qmodule/qavgpool2d.py | 50 + linger/quant/ops/qmodule/qbatchnorm1d.py | 87 + linger/quant/ops/qmodule/qbatchnorm2d.py | 80 + linger/quant/ops/qmodule/qconv1d.py | 38 + linger/quant/ops/qmodule/qconv2d.py | 38 + linger/quant/ops/qmodule/qconvbn1d.py | 129 + linger/quant/ops/qmodule/qconvbn2d.py | 129 + linger/quant/ops/qmodule/qconvtranspose1d.py | 49 + linger/quant/ops/qmodule/qconvtranspose2d.py | 51 + linger/quant/ops/qmodule/qembedding.py | 55 + linger/quant/ops/qmodule/qglu.py | 48 + linger/quant/ops/qmodule/qgru.py | 545 +++ linger/quant/ops/qmodule/qlayernorm.py | 137 + linger/quant/ops/qmodule/qlinear.py | 37 + linger/quant/ops/qmodule/qlstm.py | 578 +++ linger/quant/ops/qmodule/qmaxpool1d.py | 65 + linger/quant/ops/qmodule/qmaxpool2d.py | 65 + linger/quant/ops/qmodule/qmodule.py | 208 + linger/quant/ops/qmodule/qrelu.py | 44 + linger/quant/ops/qtensor/__init__.py | 9 + linger/quant/ops/qtensor/qadd.py | 64 + linger/quant/ops/qtensor/qbmm.py | 55 + linger/quant/ops/qtensor/qcat.py | 25 + linger/quant/ops/qtensor/qmatmul.py | 23 + linger/quant/ops/qtensor/qmul.py | 32 + linger/quant/ops/qtensor/qsigmoid.py | 82 + linger/quant/ops/qtensor/qsoftmax.py | 97 + linger/quant/ops/qtensor/qtanh.py | 82 + linger/quant/ops/qtensor/qtensor_mod.py | 154 + linger/quant/ops/qtensor/qtensor_ops.py | 372 ++ linger/quant/qtensor.py | 127 + linger/quant/quantizer.py | 304 ++ linger/tools/__init__.py | 3 - linger/tools/fix_dequant.py | 99 - linger/tools/onnx_quant/__init__.py | 1 - .../onnx_quant/fix_quant_onnx_replace.py | 50 - linger/tools/onnx_quant/transform/__init__.py | 0 .../tools/onnx_quant/transform/onnx_utils.py | 73 - .../tools/onnx_quant/transform/replace_add.py | 14 - .../transform/replace_avgpoolint.py | 13 - .../onnx_quant/transform/replace_clip.py | 27 - .../onnx_quant/transform/replace_conv.py | 139 - .../onnx_quant/transform/replace_iqsigmoid.py | 14 - .../onnx_quant/transform/replace_linearint.py | 119 - linger/tools/weight_bias_analyse.py | 48 - linger/utils.py | 288 +- requirements.txt | 17 +- setup.py | 72 +- test/example_iqtensor_dump_fuction.py | 58 - ...mple_torchintx_functional_dump_function.py | 55 - test/image_preprocess.py | 31 - test/test_Layernorm_int.py | 137 - test/test_add.py | 114 + test/test_ahead_relu.py | 109 - test/test_alexnet.py | 404 ++ test/test_avg_pool_int.py | 69 - test/test_batchnorm_int.py | 80 - test/test_bmm.py | 114 + test/test_bmm_quant.py | 59 - test/test_cat.py | 110 + test/test_clamp_batchnorm2d.py | 67 - test/test_clamp_conv1d.py | 16 - test/test_clamp_conv2d.py | 14 - test/test_clamp_convtranspose2d.py | 17 - test/test_clamp_gru.py | 159 - test/test_clamp_gru_in_onnxinfer.py | 30 - test/test_clamp_layernorm.py | 65 - test/test_clamp_linear.py | 16 - test/test_clamp_lstm.py | 118 - test/test_clamp_lstm_onnx_export.py | 63 - test/test_conv1d.py | 127 + test/test_conv2d.py | 126 + test/test_conv_dropout.py | 57 - test/test_conv_linear.py | 52 - test/test_conv_permute.py | 58 - test/test_conv_scale.py | 55 - test/test_conv_with_eval_normalize.py | 54 - test/test_conv_with_normalize.py | 56 - test/test_convbn1d_normalize.py | 81 - test/test_convbn_normalize.py | 59 - test/test_convtranspose1d.py | 123 + test/test_convtranspose2d.py | 122 + test/test_debug.py | 80 + test/test_embedding.py | 136 + test/test_embedding_int.py | 47 - test/test_export_dict_onnx.py | 86 - .../test_full_training_process_with_fusebn.py | 95 - ...g_process_with_fusebn_with_castor_quant.py | 110 - test/test_glu.py | 122 + test/test_group_conv1d_fused_bn_demo.py | 70 - test/test_group_conv_fused_bn_demo.py | 77 - test/test_gru.py | 141 + test/test_gru_int.py | 120 - test/test_iq_auto_grad.py | 375 -- test/test_iq_onnx_export.py | 488 --- test/test_iqadd_load_state_dict.py | 72 - test/test_iqdiv.py | 86 - test/test_iqmul.py | 54 - test/test_iqsigmoid_net.py | 56 - test/test_iqsihft_quant.py | 50 - test/test_iqsum.py | 76 - test/test_iqtensor.py | 465 --- test/test_iqtensor_functional.py | 50 - test/test_iqvar_net.py | 53 - test/test_layernorm.py | 118 + test/test_linear.py | 121 + test/test_lstm.py | 141 + test/test_lstm_int.py | 123 - test/test_matmul.py | 112 + test/test_maxpool.py | 120 + test/test_mul.py | 108 + test/test_ops.py | 250 -- test/test_quant_histogram.py | 65 - test/test_quant_tensor_load_state_dict.py | 66 - test/test_quant_tensor_paramter.py | 124 - test/test_relu.py | 118 + test/test_relu6_int.py | 67 - test/test_replace_float_module.py | 68 - test/test_replace_param.py | 54 - test/test_s_conv_linear_iqcat.py | 61 - test/test_s_conv_linear_normalize.py | 58 - test/test_s_normalize_init.py | 67 - test/test_scope.py | 88 - test/test_shuffle_channel.py | 66 - test/test_sigmoid.py | 108 + test/test_simple_test.py | 61 - test/test_softmax.py | 108 + test/test_softmax_int.py | 139 +- test/test_state_dict.py | 71 - test/test_tanh.py | 108 + test/test_tool.py | 56 - test/test_tqtcuda_time.py | 221 ++ test/test_trace_convtranspose.py | 193 - 261 files changed, 14807 insertions(+), 24309 deletions(-) delete mode 100644 .dockerignore create mode 100644 .vscode/settings.json delete mode 100644 README_en.md delete mode 100644 doc/image/bmm_int.png delete mode 100644 doc/image/contact_me_qr.png delete mode 100644 doc/image/conv_fused_bn.png delete mode 100644 doc/image/gru_int.png delete mode 100644 doc/image/gru_int_with_batch_length.png delete mode 100644 doc/image/gru_int_with_batch_length_with_state.png delete mode 100644 doc/image/lnn_flow_path.png delete mode 100644 doc/image/lstm_int.png delete mode 100644 doc/image/lstm_int_with_batch_length.png delete mode 100644 doc/image/lstm_int_with_batch_length_and_state.png delete mode 100644 doc/image/trace_layer_normailize_init.png rename doc/tutorial/{from_mode_to_onnx.md => export_onnx.md} (92%) delete mode 100644 doc/tutorial/get_started_for_two_stage.md delete mode 100644 doc/tutorial/install_bugs.md create mode 100644 doc/tutorial/install_errors.md create mode 100644 doc/tutorial/quant_advanced_guide.md create mode 100644 doc/tutorial/quant_quick_strat.md delete mode 100644 doc/tutorial/two_stage_quant_aware_train.md delete mode 100644 doc/tutorial/wb_analyse_tool_and_onnx_export_debug_tool.md create mode 100644 linger/checker/__init__.py create mode 100644 linger/checker/float_ops_mapper.py create mode 100644 linger/checker/iq_ops_mapper.py create mode 100644 linger/checker/onnxrunner.py create mode 100644 linger/checker/utils.py create mode 100644 linger/constrain/SparifyFFN.py create mode 100644 linger/constrain/__init__.py create mode 100644 linger/constrain/cbatchnorm2d.py create mode 100644 linger/constrain/cconv1d.py create mode 100644 linger/constrain/cconv2d.py create mode 100644 linger/constrain/cconvbn1d.py create mode 100644 linger/constrain/cconvbn2d.py create mode 100644 linger/constrain/cconvtranspose1d.py create mode 100644 linger/constrain/cconvtranspose2d.py create mode 100644 linger/constrain/cembedding.py create mode 100644 linger/constrain/clayernorm.py create mode 100644 linger/constrain/clinear.py create mode 100644 linger/constrain/cmodule.py create mode 100644 linger/constrain/cutils.py delete mode 100644 linger/conv_bn_fuser.py delete mode 100644 linger/dumper.py create mode 100644 linger/kernel/cpu/arcs_qsigmoid_kernel.cpp create mode 100644 linger/kernel/cpu/arcs_qsoftmax_kernel.cpp create mode 100644 linger/kernel/cpu/arcs_qtanh_kernel.cpp create mode 100644 linger/kernel/cpu/extension.cpp create mode 100644 linger/kernel/cpu/qlayernorm_kernel.cpp create mode 100644 linger/kernel/cpu/util_kernel.cpp create mode 100644 linger/kernel/cpu/venusa_qsigmoid_kernel.cpp create mode 100644 linger/kernel/cpu/venusa_qsoftmax_kernel.cpp create mode 100644 linger/kernel/cpu/venusa_qtanh_kernel.cpp create mode 100644 linger/kernel/gpu/arcs_qsigmoid_kernel.cu create mode 100644 linger/kernel/gpu/arcs_qsoftmax_kernel.cu create mode 100644 linger/kernel/gpu/arcs_qtanh_kernel.cu create mode 100644 linger/kernel/gpu/fake_quant_kernel.cu create mode 100644 linger/kernel/gpu/qlayernorm_kernel.cu create mode 100644 linger/kernel/gpu/util_kernel.cu create mode 100644 linger/kernel/gpu/venusa_qsigmoid_kernel.cu create mode 100644 linger/kernel/gpu/venusa_qsoftmax_kernel.cu create mode 100644 linger/kernel/gpu/venusa_qtanh_kernel.cu delete mode 100644 linger/layer_normalizer.py delete mode 100644 linger/lib/lingerext.cpython-36m-x86_64-linux-gnu.so delete mode 100644 linger/lib/lingerext.cpython-37m-x86_64-linux-gnu.so delete mode 100644 linger/lib/lingerext.cpython-38-x86_64-linux-gnu.so delete mode 100644 linger/modules/__init__.py delete mode 100644 linger/modules/modules_configs.py delete mode 100644 linger/modules/normalize_batchnorm2d.py delete mode 100644 linger/modules/normalize_conv1d.py delete mode 100644 linger/modules/normalize_conv2d.py delete mode 100644 linger/modules/normalize_convTranspose2d.py delete mode 100644 linger/modules/normalize_convTransposebn2d.py delete mode 100644 linger/modules/normalize_convbn1d.py delete mode 100644 linger/modules/normalize_convbn2d.py delete mode 100644 linger/modules/normalize_embedding.py delete mode 100644 linger/modules/normalize_fastGRU.py delete mode 100644 linger/modules/normalize_fastLSTM.py delete mode 100644 linger/modules/normalize_layernorm.py delete mode 100644 linger/modules/normalize_linear.py delete mode 100644 linger/onnx/scope.py delete mode 100644 linger/ops/__init__.py delete mode 100644 linger/ops/avgpool2d_int.py delete mode 100644 linger/ops/batchnorm_int.py delete mode 100644 linger/ops/bmm_int.py delete mode 100644 linger/ops/conv1d_int.py delete mode 100644 linger/ops/conv_int.py delete mode 100644 linger/ops/convtranspose_int.py delete mode 100644 linger/ops/embedding_int.py delete mode 100644 linger/ops/gru_int.py delete mode 100644 linger/ops/iqtensor.py delete mode 100644 linger/ops/layernorm_int.py delete mode 100644 linger/ops/linear_int.py delete mode 100644 linger/ops/linger_functional.py delete mode 100644 linger/ops/lstm_int.py delete mode 100644 linger/ops/module_self.py delete mode 100644 linger/ops/ops.py delete mode 100644 linger/ops/ops_configs.py delete mode 100644 linger/ops/ops_names.py delete mode 100644 linger/ops/relu6_int.py delete mode 100644 linger/ops/requant.py delete mode 100644 linger/ops/scaledround_int.py delete mode 100644 linger/ops/shuffle_channel.py delete mode 100644 linger/quant.py create mode 100644 linger/quant/__init__.py create mode 100644 linger/quant/calibrate_funs.py create mode 100644 linger/quant/ops/__init__.py create mode 100644 linger/quant/ops/qconfig.py create mode 100644 linger/quant/ops/qmodule/__init__.py create mode 100644 linger/quant/ops/qmodule/qSparifyFFN.py create mode 100644 linger/quant/ops/qmodule/qavgpool1d.py create mode 100644 linger/quant/ops/qmodule/qavgpool2d.py create mode 100644 linger/quant/ops/qmodule/qbatchnorm1d.py create mode 100644 linger/quant/ops/qmodule/qbatchnorm2d.py create mode 100644 linger/quant/ops/qmodule/qconv1d.py create mode 100644 linger/quant/ops/qmodule/qconv2d.py create mode 100644 linger/quant/ops/qmodule/qconvbn1d.py create mode 100644 linger/quant/ops/qmodule/qconvbn2d.py create mode 100644 linger/quant/ops/qmodule/qconvtranspose1d.py create mode 100644 linger/quant/ops/qmodule/qconvtranspose2d.py create mode 100644 linger/quant/ops/qmodule/qembedding.py create mode 100644 linger/quant/ops/qmodule/qglu.py create mode 100644 linger/quant/ops/qmodule/qgru.py create mode 100644 linger/quant/ops/qmodule/qlayernorm.py create mode 100644 linger/quant/ops/qmodule/qlinear.py create mode 100644 linger/quant/ops/qmodule/qlstm.py create mode 100644 linger/quant/ops/qmodule/qmaxpool1d.py create mode 100644 linger/quant/ops/qmodule/qmaxpool2d.py create mode 100644 linger/quant/ops/qmodule/qmodule.py create mode 100644 linger/quant/ops/qmodule/qrelu.py create mode 100644 linger/quant/ops/qtensor/__init__.py create mode 100644 linger/quant/ops/qtensor/qadd.py create mode 100644 linger/quant/ops/qtensor/qbmm.py create mode 100644 linger/quant/ops/qtensor/qcat.py create mode 100644 linger/quant/ops/qtensor/qmatmul.py create mode 100644 linger/quant/ops/qtensor/qmul.py create mode 100644 linger/quant/ops/qtensor/qsigmoid.py create mode 100644 linger/quant/ops/qtensor/qsoftmax.py create mode 100644 linger/quant/ops/qtensor/qtanh.py create mode 100644 linger/quant/ops/qtensor/qtensor_mod.py create mode 100644 linger/quant/ops/qtensor/qtensor_ops.py create mode 100644 linger/quant/qtensor.py create mode 100644 linger/quant/quantizer.py delete mode 100644 linger/tools/__init__.py delete mode 100644 linger/tools/fix_dequant.py delete mode 100644 linger/tools/onnx_quant/__init__.py delete mode 100644 linger/tools/onnx_quant/fix_quant_onnx_replace.py delete mode 100644 linger/tools/onnx_quant/transform/__init__.py delete mode 100644 linger/tools/onnx_quant/transform/onnx_utils.py delete mode 100644 linger/tools/onnx_quant/transform/replace_add.py delete mode 100644 linger/tools/onnx_quant/transform/replace_avgpoolint.py delete mode 100644 linger/tools/onnx_quant/transform/replace_clip.py delete mode 100644 linger/tools/onnx_quant/transform/replace_conv.py delete mode 100644 linger/tools/onnx_quant/transform/replace_iqsigmoid.py delete mode 100644 linger/tools/onnx_quant/transform/replace_linearint.py delete mode 100644 linger/tools/weight_bias_analyse.py delete mode 100644 test/example_iqtensor_dump_fuction.py delete mode 100644 test/example_torchintx_functional_dump_function.py delete mode 100644 test/image_preprocess.py delete mode 100644 test/test_Layernorm_int.py create mode 100644 test/test_add.py delete mode 100644 test/test_ahead_relu.py create mode 100644 test/test_alexnet.py delete mode 100644 test/test_avg_pool_int.py delete mode 100644 test/test_batchnorm_int.py create mode 100644 test/test_bmm.py delete mode 100644 test/test_bmm_quant.py create mode 100644 test/test_cat.py delete mode 100644 test/test_clamp_batchnorm2d.py delete mode 100644 test/test_clamp_conv1d.py delete mode 100644 test/test_clamp_conv2d.py delete mode 100644 test/test_clamp_convtranspose2d.py delete mode 100644 test/test_clamp_gru.py delete mode 100644 test/test_clamp_gru_in_onnxinfer.py delete mode 100644 test/test_clamp_layernorm.py delete mode 100644 test/test_clamp_linear.py delete mode 100644 test/test_clamp_lstm.py delete mode 100644 test/test_clamp_lstm_onnx_export.py create mode 100644 test/test_conv1d.py create mode 100644 test/test_conv2d.py delete mode 100644 test/test_conv_dropout.py delete mode 100644 test/test_conv_linear.py delete mode 100644 test/test_conv_permute.py delete mode 100644 test/test_conv_scale.py delete mode 100644 test/test_conv_with_eval_normalize.py delete mode 100644 test/test_conv_with_normalize.py delete mode 100644 test/test_convbn1d_normalize.py delete mode 100644 test/test_convbn_normalize.py create mode 100644 test/test_convtranspose1d.py create mode 100644 test/test_convtranspose2d.py create mode 100644 test/test_debug.py create mode 100644 test/test_embedding.py delete mode 100644 test/test_embedding_int.py delete mode 100644 test/test_export_dict_onnx.py delete mode 100644 test/test_full_training_process_with_fusebn.py delete mode 100644 test/test_full_training_process_with_fusebn_with_castor_quant.py create mode 100644 test/test_glu.py delete mode 100644 test/test_group_conv1d_fused_bn_demo.py delete mode 100644 test/test_group_conv_fused_bn_demo.py create mode 100644 test/test_gru.py delete mode 100644 test/test_gru_int.py delete mode 100644 test/test_iq_auto_grad.py delete mode 100644 test/test_iq_onnx_export.py delete mode 100644 test/test_iqadd_load_state_dict.py delete mode 100644 test/test_iqdiv.py delete mode 100644 test/test_iqmul.py delete mode 100644 test/test_iqsigmoid_net.py delete mode 100644 test/test_iqsihft_quant.py delete mode 100644 test/test_iqsum.py delete mode 100644 test/test_iqtensor.py delete mode 100644 test/test_iqtensor_functional.py delete mode 100644 test/test_iqvar_net.py create mode 100644 test/test_layernorm.py create mode 100644 test/test_linear.py create mode 100644 test/test_lstm.py delete mode 100644 test/test_lstm_int.py create mode 100644 test/test_matmul.py create mode 100644 test/test_maxpool.py create mode 100644 test/test_mul.py delete mode 100644 test/test_ops.py delete mode 100644 test/test_quant_histogram.py delete mode 100644 test/test_quant_tensor_load_state_dict.py delete mode 100644 test/test_quant_tensor_paramter.py create mode 100644 test/test_relu.py delete mode 100644 test/test_relu6_int.py delete mode 100644 test/test_replace_float_module.py delete mode 100644 test/test_replace_param.py delete mode 100644 test/test_s_conv_linear_iqcat.py delete mode 100644 test/test_s_conv_linear_normalize.py delete mode 100644 test/test_s_normalize_init.py delete mode 100644 test/test_scope.py delete mode 100644 test/test_shuffle_channel.py create mode 100644 test/test_sigmoid.py delete mode 100644 test/test_simple_test.py create mode 100644 test/test_softmax.py delete mode 100644 test/test_state_dict.py create mode 100644 test/test_tanh.py delete mode 100644 test/test_tool.py create mode 100644 test/test_tqtcuda_time.py delete mode 100644 test/test_trace_convtranspose.py diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 3e4e48b..0000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3bbad5f..7651e65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,5 @@ -__pycache__ -test/__pycache__ - -*.onnx -*.onnx.ignore -*.pt - -data.ignore/ - dist +data linger.egg-info linger.egg-info/dependency_links.txt linger.egg-info/PKG-INFO @@ -17,6 +9,7 @@ linger.egg-info/top_level.txt test/data.ignore test/data.ignore/aa.pt test/data.ignore/test1_baseline.pt +data.ignore/ doc/build/ examples/dump examples/dump_all @@ -24,8 +17,11 @@ examples/dump_filter examples/dump_part examples/resnet50.onnx.ignore examples/shufflenet_v2_x1_0.bin.ignore - +*.onnx +*.onnx.ignore +*.pt *.log .vscode - -build \ No newline at end of file +__pycache__ +test/__pycache__ +data.ignore diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..a8c2003 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/LICENSE b/LICENSE index ac4ade6..5f1c80a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2022 LISTENAI Authors. All Rights Reserved +Copyright (c) 2025 LISTENAI Authors. All Rights Reserved Apache License Version 2.0, January 2004 @@ -188,7 +188,7 @@ Copyright (c) 2022 LISTENAI Authors. All Rights Reserved same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright (c) 2022 LISTENAI Authors. All Rights Reserved. + Copyright (c) 2025 LISTENAI Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/MANIFEST.in b/MANIFEST.in index 540e1e3..4f7e728 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include requirements.txt include linger/* -include linger/lib/* \ No newline at end of file + +exclude linger/extension \ No newline at end of file diff --git a/README.md b/README.md index d2e3902..bead395 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,10 @@ ![linger_logo](doc/image/linger_logo.png) -------------------------------------------------------------------------------- #### [English](README_en.md) | 简体中文 - -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pylinger.svg)](https://pypi.org/project/pylinger) -[![PiPI](https://badge.fury.io/py/pylinger.svg)](https://pypi.org/project/pylinger/) -[![License](https://img.shields.io/github/license/LISTENAI/thinker.svg?style=flat-square)](https://github.com/LISTENAI/linger/blob/main/LICENSE) -[![linux](https://github.com/LISTENAI/linger/actions/workflows/auto_test.yml/badge.svg)](https://github.com/LISTENAI/linger/actions/workflows/auto_test.yml) -linger是由安徽聆思科技和合肥智能语音公司联合开源的的神经网络量化训练组件，是聆思科技开源的AI生态工具链LNN(ListenAI Neural Network)的一部分，结合另一个聆思开源的推理引擎框架[thinker](https://github.com/LISTENAI/thinker)可实现产业级深度学习训练推理一体化平台，集深度学习量化训练和引擎推理、LUNA器件库和丰富的辅助组件于一体。LNN是专为聆思AIOT芯片（目前只支持CSK60xx系列）定制开发，助力开发者轻松在聆思VENUS芯片上快速上线AI业务，帮助越来越多嵌入式尤其是AIOT产品实现AI赋能，助力产业智能化升级。目前LNN工具链已支持聆思芯片在计算机视觉、语音唤醒、语音识别、离线翻译等10多个AI应用场景中的产品落地。 ## 方案简介 -linger基于PyTorch对聆思LUNA系列芯片进行深度定制，在神经网络训练的前向过程中将激活和权重量化到8bit，通过参数调整得到量化无损的8bit模型 +linger基于PyTorch对聆思IOT系列芯片进行深度定制，在神经网络训练的前向过程中将激活和权重量化到8bit，通过参数调整得到量化无损的8bit模型 ![doc/image/solution.png](doc/image/solution.png) @@ -21,83 +15,21 @@ linger 是基于 PyTorch 的量化方案，在原始浮点训练代码中加入 ### 2. 拓展性好 linger 基于 PyTorch 进行量化算子的搭建，因此只要符合 PyTorch 拓展算子的规范，你可以添加任何量化算子到 linger 中来完成你的量化需求 -### 3. 工具链完整 -linger 后端适配 [thinker](https://github.com/LISTENAI/thinker) 推理引擎，thinker 推理引擎为CSK60XX而生，功能完善，量化训练与推理过程可无缝衔接，同时训练推理二进制一致得到保证 - ## 快速入门 - [安装](doc/tutorial/install.md)：支持pip、源码、docker三种安装方式 -- [浮点-定点两阶段量化训练](doc/tutorial/get_started_for_two_stage.md): 先进行浮点网络的约束训练，再针对量化友好的浮点模型进行量化训练微调 -- [浮点-定点两阶段量化训练方案详解](doc/tutorial/two_stage_quant_aware_train.md) -- [onnx导出教程](doc/tutorial/from_mode_to_onnx.md)：将量化无损的PyTorch模型导出为ONNX格式的模型 -- [权重分析工具使用及量化onnx导出错误调试](doc/tutorial/wb_analyse_tool_and_onnx_export_debug_tool.md) - -## 工程示例 -AI算法落地基本涵盖六个阶段：模型规约性检查、浮点训练、量化训练、模型打包、模拟引擎执行、固件烧录并芯片运行。其中固件烧录并芯片运行需要在聆思的开发板上来完成，如有需要请与我们联系，这里不做进一步介绍。其它五个阶段的流程示例图如下： -![lnn_flow_path](doc/image/lnn_flow_path.png) -其中模型规约性检查的功能是穿插在量化训练和模型打包中来完成的。 -我们先假设模型结构与底层硬件完全适配，介绍流程中各个阶段，再介绍模型规约性检查的具体实现（实际开发过程中规约性检查要在模型结构初步进行，避免后续工作返工）。 -### 1. 浮点训练 - 我们基于[pytorch-cifar100](https://github.com/weiaicunzai/pytorch-cifar100)来进行功能展示 - 首先确保在当前环境下（建议linger-env)，浮点模型训练基于pytorch能够跑起来。 -```Shell -python train.py -net resnet50 -gpu -``` - 建议采用两阶段量化训练，对浮点训练的数据进行范围约束，只需[添加少量代码](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/resnet_modify1.md). - 为避免冲突，将tesnorboard[功能关闭](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/resnet_modify2.md)。同样的指令开启训练，运行几个epoch后，在checkpoint/resnet50文件夹中生成了一个**.pth文件 - -### 2. 量化训练和导出 - 加载步1中保存的浮点模型**.pth，[修改约束代码](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/linger_set2.png)，即可将浮点算子替换为量化算子。同样的指令开启量化训练，训练几个epoch后，同样在checkpoint/resnet50文件夹中生成了一个**.pth文件。 - 使用linger的模型转换工具，将[模型转换成onnx计算图](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/onnx_export.png)。 - -### 3. 模型分析和打包 - 切换到thinker-env环境，使用thinker离线工具tpacker对步2生成的onnx计算图打包，这里我们以训练好的resnet18模型为例，进行打包 -```Shell -tpacker -g demo/resnet18/resnet18-12-regular.onnx -d True -o demo/resnet18/model.bin -``` -这里使用到的资源可以从[thinker/demo/resnet18](https://github.com/LISTENAI/thinker/tree/main/demo/resnet18)中获取 - -### 4. 推理执行 - 使用调用示例工程test_thinker，指定输入数据、资源文件和输出文件名称即可运行模拟代码。 -```Shell -chmod +x ./bin/test_thinker -./bin/test_thinker demo/resnet18/input.bin demo/resnet18/model.bin demo/resnet18/output.bin 3 32 32 6 -``` - 注意：推理执行需要[安装thinker源码](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/install.md)，并完成编译。 - - -### 5. 规约性检查 - 该阶段不关注模型的效果，只关注模型的结构是否和底层硬件相适配，功能实现贯穿了1~4步 - * 在步1中，对模型参数进行初始化或者训练几个epoch即可将模型文件导出，无需模型收敛。 - * 步2中加载步1的模型文件，进行量化训练时，会对算子参数的合规性进行检查，如有不符合的设置，报错退出[错误示例](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/resnet50_linger_err.png)。用户根据报错信息修改层参数并返回步1，直至通过步2。 - * 步3中加载步2的计算图，工具会对节点的tensor大小进行检查，[如果tensor大小超限会报错退出](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_err.png)。否则进入内存分析阶段，会在根目录下生成[内存分析报告](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_Mem1.png)，并提示整体的flash/psram/share-memory占用。对于超过硬件限制的报错，用户可结合报错信息和[内存分析报告](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_Mem2.png)来定位计算图中的超限的算子，返回步1进行模型结构调整，直至[通过步3的打包流程](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_sucess.png)。 - 至此完成模型规约性检查，确保了模型能够在芯片上能够跑起来。模型效率评估目前只支持在芯片上部署运行，具体需求可联系我们。 +- [量化训练快速入门](doc/tutorial/quant_quick_strat.md): 先进行浮点网络的约束训练，再针对量化友好的浮点模型进行量化训练微调 +- [量化训练进阶指导](doc/tutorial/quant_advanced_guide.md): 量化进阶配置 +- [onnx导出教程](doc/tutorial/export_onnx.md)：将量化无损的PyTorch模型导出为ONNX格式的模型 ## 能力展示 - [linger API](doc/tutorial/linger_api.md) -- [支持量化OP列表](doc/tutorial/support_quant_ops.md)及[模型结构限制说明](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/restrain_of_model.md) +- [支持量化OP列表](doc/tutorial/support_quant_ops.md) ## 常见问题 -- [安装出现问题解决](doc/tutorial/install_bugs.md) +- [安装出现问题解决](doc/tutorial/install_errors.md) - [量化常见问题与注意事项](doc/tutorial/quant_faq.md) ## 版本说明 - 请参考[RELEASE](doc/tutorial/release.md) -## 交流与反馈 -- 欢迎您通过 Github Issues 来提交 BUG 与建议 -- 技术交流微信群 -![concat us](doc/image/contact_me_qr.png) - -## 引用 -- [PyTorch](https://github.com/pytorch/pytorch) -- [ONNX](https://github.com/onnx/onnx) -- [pytorch-cifar100](https://github.com/weiaicunzai/pytorch-cifar100) -- -## 应用示例 -* 鼾声检测[https://github.com/mywang44/snoring_net] -* 离线翻译[https://github.com/dwzhang00/Offline-translation] -* 二维码检测与识别[https://github.com/mywang44/YOLOv1_QRcode_Detection] - -## 版权和许可证 -- linger 由 [Apache-2.0 license](LICENSE) 提供 diff --git a/README_en.md b/README_en.md deleted file mode 100644 index 1650dbb..0000000 --- a/README_en.md +++ /dev/null @@ -1,105 +0,0 @@ -![linger_logo](doc/image/linger_logo.png) --------------------------------------------------------------------------------- -#### English | [Chinese](README.md) - -[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pylinger.svg)](https://pypi.org/project/pylinger) -[![PiPI](https://badge.fury.io/py/pylinger.svg)](https://pypi.org/project/pylinger/) -[![License](https://img.shields.io/github/license/LISTENAI/thinker.svg?style=flat-square)](https://github.com/LISTENAI/linger/blob/main/LICENSE) -[![linux](https://github.com/LISTENAI/linger/actions/workflows/auto_test.yml/badge.svg)](https://github.com/LISTENAI/linger/actions/workflows/auto_test.yml) - -Linger is an open source neural network quantization training component by LISTENAI, designed for use with the AIOT chip CSK60XX. This component combines Linger's open source high-performance neural network inference framework -[thinker](https://github.com/LISTENAI/thinker) can achieve training and inference integration, helping AI developers to quickly give business with AI capabilities based on CSK chip. Currently linger + thinker tool chain has supported the use of CSK chip in more than 10 AI application scenarios such as computer vision, voice wakeup, speech recognition, offline translation, etc. - - -## Introduction -The linger is based on PyTorch to deeply customize the LISTENAI LUNA series chip, quantize the activation and weight to 8bit in the forward process of neural network training, and get the quantized lossless 8bit model by parameter adjustment. - -![doc/image/solution.png](doc/image/solution.png) - -## Technical Highlights -### 1. High Ease of Use -linger is a PyTorch-based quantization scheme. Adding one line of linger-related code to the original floating-point training code can complete the replacement of quantization operators, and the quantization training can be completed using the original training process without other complicated settings. - -### 2. Good Scalability -linger is based on PyTorch to build quantization operators, so you can add any quantization operator to linger to complete your quantization needs as long as it meets the specifications of PyTorch extension operators. - -### 3. Complete Toolchain -The backend is adapted to [thinker](https://github.com/LISTENAI/thinker) inference engine, thinker inference engine for CSK60XX, which is fully functional and seamlessly integrates quantization training and inference process, while the binary consistency of training and inference is guaranteed. - - -## Quick Start -1. [Installation](doc/tutorial/install.md)：support pip, source code, docker and other installation methods -2. [Floating-point-fixed-point two-stage quantization training](doc/tutorial/get_started_for_two_stage.md): first the constraint training of floating-point network, and then the quantization training fine-tuning for the quantization-friendly floating-point model -3. [ONNX export tutorial](doc/tutorial/from_mode_to_onnx.md)：exporting quantized lossless PyTorch models to ONNX format -4. [Complete introductory examples](examples/)：provide several newbie-friendly introductory quantization examples - -## Demo -The implementation of AI algorithms basically covers six stages: model specification check, floating-point training, quantization training, model packaging, simulation engine execution, firmware burning and chip operation. The firmware programming and chip operation need to be completed on the development board of Lenses. If necessary, please contact us, and no further introduction will be made here. The flow chart of the other five stages is as follows: -![lnn_flow_path](doc/image/lnn_flow_path.png) -Among them, the function of model regularity check is interspersed in quantization training and model packaging. -We first assume that the model structure is fully compatible with the underlying hardware, introduce each stage in the process, and then introduce the specific implementation of the model convention check (in the actual development process, the convention check should be carried out initially on the model structure to avoid rework in subsequent work). -### 1. Floating-point training -We are based on [pythoch-cifar100](https://github.com/weiaicunzai/pytorch-cifar100) for function demonstration -First of all, Make sure that in the current environment, the floating-point model training can run based on pytorch. -```Shell -python train.py -net resnet50 -gpu -``` -It is recommended to use two-stage quantization training to restrict the range of floating-point training data, and only need to [add a small amount of code](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/resnet_modify1.md). -To avoid conflicts, turn tesnorboard[function off](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/resnet_modify2.md). Start the training with the same command, and after running several epochs, a **.pth file is generated in the checkpoint/resnet50 folder - -### 2. Quantization training and Export -Load the floating-point model **.pth saved in step 1, and [modify the constraint code](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/linger_set2.png) to replace the floating-point operator with a quantized operator. The same command starts quantization training. After several epochs are trained, a **.pth file is also generated in the checkpoint/resnet50 folder. -Use linger's model conversion tool to [convert the model into an onnx calculation graph](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/onnx_export.png). - -### 3. Model analysis and packaging -Use the thinker offline tool tpacker to pack the onnx calculation graph generated in step 2 -```Shell -tpacker -g demo/resnet28/resnet18-12-regular.onnx -d Ture -o demo/resnet28/model.bin -``` -we can acquire resource from [thinker/demo/resnet18](https://github.com/LISTENAI/thinker/demo/resnet18/) -### 4. Engine Execution -Use the sample project test_thinker to run the simulation code by specifying the input data, resource file and output file name. -```Shell -chmod +x ./bin/test_thinker -./bin/test_thinker demo/resnet28/input.bin demo/resnet28/model.bin demo/resnet28/output.bin 3 32 32 6 -``` -Simplify the overall processing process here, with the engine input being a normalized 3x32x32 image and the output taking max_ The ID corresponding to value is used as the classification result. The processing of input images can refer to the [Image Processing Script](tools/image_process.py), or the processed test set images can be taken from Pytorch cifar100 for testing. - -### 5. Conventional check -At this stage, we do not pay attention to the effect of the model, but only pay attention to whether the structure of the model is compatible with the underlying hardware, and the function realization runs through steps 1~4 -* In step 1, the model file can be exported by initializing the model parameters or training a few epochs without model convergence. -* Load the model file of step 1 in step 2. When performing quantitative training, the compliance of operator parameters will be checked. If there are any settings that do not meet the requirements, an error will be reported and exit -[error example](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/resnet50_linger_err.png). The user modifies the layer parameters according to the error message and returns to step 1 until step 2 is passed. -* Load the calculation graph of step 2 in step 3, the tool will check the tensor size of the node, [if the tensor size exceeds the limit, an error will be reported and exit](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_err.png). Otherwise, enter the memory analysis stage, and generate a [memory analysis report](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_Mem1.png) in the root directory, and prompt the overall flash /psram/share-memory occupied. For errors that exceed the hardware limit, users can combine the error information and [Memory Analysis Report](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_Mem2.png) to locate the calculation graph The overrun operator returns to step 1 to adjust the model structure until [through the packaging process of step 3](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/images/Resnet50_sucess.png ). -So far, the model compliance check has been completed, ensuring that the model can run on the chip. Model efficiency evaluation currently only supports deployment and operation on chips, please contact us for specific needs. - -## Quantitative Advancement - - [Floating-point-fixed-point two-stage quantization training program detailed explanation](doc/tutorial/two_stage_quant_aware_train.md) - - [Use of weight analysis tools and debugging of quantitative onnx export errors](doc/tutorial/wb_analyse_tool_and_onnx_export_debug_tool.md) - -## Frequently Asked Questions -- [Installation problem solving](doc/tutorial/install_bugs.md) -- [Quantification of common problems and notes](doc/tutorial/quant_faq.md) - -## Release Not -- Please refer to [RELEASE](doc/tutorial/release.md) - -## Data Search -- [linger API](doc/tutorial/linger_api.md) -- [List of supported quantization OPs](doc/tutorial/support_quant_ops.md) and [their restrictions](https://github.com/LISTENAI/thinker/blob/main/thinker/docs/tutorial/restrain_of_model.md) - -## Communication and Feedback -- You are welcome to submit bugs and suggestions via Github Issues -- Technical Communication WeChat Group -![concat us](doc/image/contact_me_qr.png) - -## Reference -- [PyTorch](https://github.com/pytorch/pytorch) -- [ONNX](https://github.com/onnx/onnx) -- [pytorch-cifar100](https://github.com/weiaicunzai/pytorch-cifar100) -- -## Applications -* snoring detect[https://github.com/mywang44/snoring_net] - -## License -- linger is provided by the [Apache-2.0 license](LICENSE) diff --git a/doc/image/bmm_int.png b/doc/image/bmm_int.png deleted file mode 100644 index 0a9e8f4560248493daf7531397a106ab2d2bcb86..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21929 zcmb@ubzD^6w>OSbN~%bwqM{((4I(Ao(jw9gLzjY-G)OaaBPHDe3Zpbb!wfaVfDGOB zoIyYL`9AmF@BKZm-|zPao;fr7?7jBhYp?ZQ@3oF$s>-s2cvN^87#M`|axc{}FmCJu zf0nm!fPZIfk{B3O(fH>k*ued5M>$;=3=Emi-MEf-Kl3wp-U}3X4|&D0aF^l6+go_OHsqbZgyPJx z{qPimN$Dk;-?wH-yjHyv*~!(1ce5@B>qBU?6)53p>wCOl-B=22*&Ez85}ITJ`m16k z_F9>!VpKV*B=uD6xNyU%w4ekfdN8&D7nblH=N=S^r&NcXWDh%C2n)ysO66+q1*Z&W zcc?ZQeMsLb%!`!6|Lr6G*zTop%kg4(SnN-*4pZljntzKy`AJvH@Bu^_gy-Cr8Rzzt zHD>SPbB&b7-N=(X%cea{Oib3W(j;TQlqkU!ifPvgLn|Kx<6+mp8WQm0W)B9fGvq;y z?G)GP)SzDhpNFlj;^zGJ9jpifE zl|Q-%n9kAaC~6kA;6ly^WMkcOH1tsCry&*Dyrz-DcF30_METg)lP{wDy)Y=Ek?Z8( z;lmT^8MVwXu(3j;dUWe;yWZE!xpc*?mXE-^?|H9Ie9<)xZi=sw+C&du1kMz8FZZ}T zLo{RGTsGZU3`|%dy+=0!Xq@9_G)>u?m*U``GR7E*Ocro<&m;LAykXtO^(c<-A?xS>F)?vzy70!A>QS6Sbrk*TBqAy9JiSfZTxIK$Q%Cqa zOX=-b)1fur297jFWb&j@Zbo0fDE}NBnE17x#P8Lat66s16&MH)8u&=(fQtas1=u!* z)w5?-A&cLLd5)9#Q;qeC1l;M29DmO&H2nEx;T+l*SVW{$o? zYeI(ap>F)c47BP2eUZZ7r~2>2;RN-cXu79w&7GKe+Y zOu3BG$+efYzG(k&Mjw#&;ig0(!SBj0BG7tq;mBtFT=|nq4=E`JUD8w11hXVVw;gd2 zd-MuQLTOgrH%j{};a06n@usKs8uHY4lPt?!0~>H%R{A)FpLbE#$Nzxu=jpdweH8IS+wyY?c!qde%6- z^C+x#DTHo%swW^0J-g2?;E~FuGdj@w;L#Zn?FnmyBeQZ=B!A zriLEi`Ag(wc9cek>3R5dMJWhRMkwE(Zsdz~%hE_i9WS?-4vM}!FU|sM)9?88jGr!n z<)*WFEGKs^Rt65-zdY_?!1S&Jxya?-<16wKWX7K6x*?zrcWIM*(1jF*uBpGKn;lJYn2>5LHlke>dayyHH5 zsTT}3=^k(wCvXd60hYUyy3eUXWSV-##x=_7Y`|c!Gf z$&)O1Gx(aX{PRatGcScgLP?Xr0(Fd}YhaH1%UaDh!AtLH08OKPg^BQJ2ac+NN1#IB zc~*zWJ564)jg^8&9vF9V|NKPteR^lm7v)mpxbWgi*%sd72^y9rAI2mWd87V439Yk0 zPrvC`!tcx&eYr^25Y=@})!2CKJGbMfFpbR7fGDz;uF`j$;ayrz*Uk1h?oZu9oJxxH zJNy(Jp?XOtnMZ!-o!1S?0`!GA78O;E!6o{{C%cC}n<%mOGAIk&(>3c=+jNJ9U|nN6 zux~H~Mm#oxlLD8DJC9+Kk3ot)`Wc-Zhn13XM{2+d_K+27nH}61-eNcVjl{X?KF)Rl_59w;oNLo&9J6Kg)TO2lxxsUmFM-Xk<0rv zBt5F)^8Fe2czwd0MrU$~odH%s@48o9x0rFX0SaCivFf1!zxoy7$H~fu$WydUYI!Fl z?yYlxHS=Ap+9D5=?e1m`!>f^c?=U0C;}fL6x-)kkX`O6}K*P~mSqzZ}nZ)1u<~+y$ zKCmy0YLN2fAH{2%PArf`76LNhoIPXnU+#3h*TDLI9wd&cIUpGTp*%irkWKG|m6Yjg z>OcddfVCG~Svuy=JUcIDMRU!-@gD5^4n%9PI|;`mQT42T!3C zGmRvoRq7YxnKlNp?T^f4h9&ck?mYYT6@v}YR8SyH!?~0z6FTw3IQ2Y#T`7@HM}7tulb07Z1Wgl_Ca~z78de_WG7i{3 z-UJQH9~z`e$_tlKJ2IDsKi20|=zFsBb?fP?4PhvVSu`aqGc|s57u% zOcDFM&yT)jw_A>tp*Hv}I^t`rpA`{wsBF#d7`+FY6@5#NLu=39>*Km5XBnY+T?{}~ ze@oV746ttEW|Yt6?XY_bv^!NVZSr|du|f9f;x_Oibnntb=+EbI*JnY%LbR?;5H0JKNW|h^9(FT^E(f;R4puK+nQoP_n@v9cW zzLTUHsB~dWp}Sv7C%g8j0%;*O#8OFXhGe|RWlNOyeIa>_6&QKoWjKp`7)ZcQ5!B~+ zXmL5#M(Zk1Jvqvp}m#_xxp6bnve4>F_*vM0^L536-yVf@NbSP zhnj|ceO5M$7AHd%hyQcc+=)-cABQ~E#`(}?{1oLxM{(O=) zq(676Ere=VT1$5s$lj=E&uBYp-5TEVI}&!81ldFn6RZ}9z?{hK7okKh&AX*|JaBr^+r+=Dze z>(N3#x3e;cjU;fwj$vm?rh_A>EJyrp)TXfb*tJeRD-zE68&13b&-i7jG+Q`I^w2JOW<QH#x!yN1c{$>d>;Rkf-gk3YF6i4{?oA3$87HhJr?VV~>o0tU%jr|L zErz|zp{2~Av$Hr}CKlg#mx1!|8-PfyCs~%I7WjA3j=10+x-~qd%#ar9a~u#Eu#I0Z zwHXs3yDEt|#Op|90V8%Fx(`%n*a>xu55X^LyoWYfRFQQN4($PKg|2~TKH=RJ+9!t* zNz3-doIZ94tGs z2Ymc(?yN%Bg3uDk=MVUg^VcEwysLiJS>beDr`@{y>jOZ4!eow9d2RYs`92ajd>?c~ zvOsy(@?NQN1AC@M>30Nk562ZJH1AK%=VG~i@UW|~736*PL&T_+S3_PL_nXm&!uW6GC@r}zV`y#^qJ~Qx&ALRVq$&Z6N$CLqZ%ynQ_Hn(ZPyPA*pi2BHU5UoE zzZ~t2BgM^C6K>Mb6_d}LLr%~Wd<-;P!cSf4c+#G!@p&N}G`_ogpjj}W9BXm9mf>--lH|5}RJ zQUZE8hhm61xTu$tN+{H|yU2Y(6Gf~bP z!g2-JM<_ozLTjaXHJ>;+I38EG1Wn3{Sv5 zAR#~+y(#?N)GKV7tMzCzYrk4{;dX64`V|V>7?)Ho_9ZMq+;UpCFtT?r?4yrXUf2!j z1rRYV-y?UW`Bp~LMyhUt!-W>_=#N~$$V7|iasQF?KZ(CfjxeZb-3dFjRGJW7HSr7T z?lPeF^#K+2>sYcclBQw+>-mFK@&Fpd^1IM)3(+9mZ;gr>mXT6$57hT)U;&C?ID(6 zsMK3YzvTp-)CYnPrhXAU(Gd=Ix-NLm$o{xvOs4C&E@-<0H;r&8H90xU!EY~Z|EG#y zoU-8Z{b~QJ3o8JU>GZbjCRF52Ra-#inZPfTjJ%$~JXgV@TZnuTQbCBM&oq%UQIss5 z=f}1)*w%9qFs%b-$!3V)epKVtU)5Z>?L6r7y<((RCmf3tiqg`5C{YFoxAt~}( zDVc`4;KN8IicABWlM_U)Xf|%istvMTMD#@NL z1S<42&x;q<&RlC->Irm=wKmd+wiY~3-{hBfmp*oA(CqgyJj}pq9INV~D>V=$JT5)} z={N)!_`lByRKOL){jI8t+em0+=wRUAkTWvP`xuv@-3p`V@?4{eZAb9B zThnc?W#3p)fOP=+lTA6|@FGPVgxnmY(V?btvXLcwla$qdu$UH$Ww53I>G$L50! z7bW6+_!Hr7Z0y|wd|o{Cc}a0~_QNK-xKD~_BZTF7jP&-&>m+!04}X`GMSG}i@2F5P zoVcH^hgGv;tRQXo4&oGi%w1yN05;zUD|KHnirKT_q9Z;PUUlC(B~5KkSEykqbssBG z7SwPG-t@U#)-gG+*QDB;4mZx{rCyn{{@L=RRP)@7vRU~6e*U9LFKVuJW4YgJo%CQj za-(_XpvaeZMrSwJ+YnbQmSGixJdnkdrgEO4`Uh01!GKuxgQ=LUiEYfx(IQ{WDDLwo zs<7zSsf2rHju#)4JcO>eT8xM^X=_NlnmjEAyAEG=d<>XPr^vDLl6)xH=5ylAS@gC0 zcMxtWYX%+KknZz9bVMZAk9I>c^GWG8=-YzV2a#slMLaB4{gb@bKK;o?-LDA6k76kG zOS^OTJU#Q7wDI<+Bx6+Qj~BjZW=S&o;_vqM^H5jPd=r8k?DN8DF2_EceTfe)G|U~D zdgHdbGdCYETj}F+2ay%}?$lZI*6uqA;OXrj`abt=JC_eU!`2#4_MX3C9$pyYs6wSH+YUzd5r~$4e%)!>xq2^WjE;>75wT|Rp zQmR&gj6n~Z_FjGYS*bT5C4OTJAm50b_*JzL&+?>X9n3>4I~T8${>D_mGOd-` zJ>+D<|FOAI{M3Y#TNBaoCvig^h2^XjjFplvb)T#&34Oj+bv3;zr9)u@#@I&4sC