From 5142544740f424c0e79dd645daa21020420f3f24 Mon Sep 17 00:00:00 2001 From: nitao Date: Fri, 20 Sep 2024 15:54:38 +0800 Subject: [PATCH 1/3] tune ap memory cache policy (#18852) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在memcache一系列比较大的改动后,ap query scan的策略需要相应修改。原来的策略删除。 现在改成每个reader,动态判断策略。当超过smallScan的阈值后,再使用SkipMemoryCacheWrites策略 经过测试,ap性能略有提升 Approved by: @zhangxu19830126, @XuPeng-SH --- pkg/pb/shard/shard.pb.go | 262 +++++++++----------- pkg/vm/engine/disttae/engine.go | 12 - pkg/vm/engine/disttae/reader.go | 6 +- pkg/vm/engine/disttae/txn_table.go | 3 - pkg/vm/engine/disttae/txn_table_sharding.go | 5 - pkg/vm/engine/disttae/types.go | 27 +- pkg/vm/engine/test/reader_test.go | 1 - proto/shard.proto | 3 +- 8 files changed, 131 insertions(+), 188 deletions(-) diff --git a/pkg/pb/shard/shard.pb.go b/pkg/pb/shard/shard.pb.go index f3515e2d1528..230663eff48f 100644 --- a/pkg/pb/shard/shard.pb.go +++ b/pkg/pb/shard/shard.pb.go @@ -1964,8 +1964,7 @@ func (m *KeyParam) GetKey() []byte { type ReaderBuildParam struct { RelData []byte `protobuf:"bytes,1,opt,name=relData,proto3" json:"relData,omitempty"` Expr *plan.Expr `protobuf:"bytes,2,opt,name=expr,proto3" json:"expr,omitempty"` - ScanType int32 `protobuf:"varint,3,opt,name=scanType,proto3" json:"scanType,omitempty"` - TombstoneApplyPolicy int32 `protobuf:"varint,4,opt,name=tombstoneApplyPolicy,proto3" json:"tombstoneApplyPolicy,omitempty"` + TombstoneApplyPolicy int32 `protobuf:"varint,3,opt,name=tombstoneApplyPolicy,proto3" json:"tombstoneApplyPolicy,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2018,13 +2017,6 @@ func (m *ReaderBuildParam) GetExpr() *plan.Expr { return nil } -func (m *ReaderBuildParam) GetScanType() int32 { - if m != nil { - return m.ScanType - } - return 0 -} - func (m *ReaderBuildParam) GetTombstoneApplyPolicy() int32 { if m != nil { return m.TombstoneApplyPolicy @@ -2392,121 +2384,120 @@ func init() { func init() { proto.RegisterFile("shard.proto", fileDescriptor_319ea41e44cdc364) } var fileDescriptor_319ea41e44cdc364 = []byte{ - // 1820 bytes of a gzipped FileDescriptorProto + // 1805 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0x4f, 0x73, 0x1b, 0x49, - 0x15, 0xf7, 0xc8, 0x23, 0x59, 0x7a, 0x92, 0xed, 0x71, 0xc7, 0xf1, 0xaa, 0x42, 0xd6, 0x88, 0xa9, - 0xec, 0x62, 0x1c, 0x62, 0x17, 0x0e, 0xcb, 0x56, 0x41, 0xf8, 0xe3, 0x48, 0xa9, 0xc4, 0x15, 0xac, + 0x15, 0xf7, 0xc8, 0x23, 0x59, 0x7a, 0x92, 0xed, 0x71, 0xc7, 0xf1, 0xaa, 0x4c, 0xd6, 0x88, 0xa9, + 0xec, 0x62, 0x1c, 0x62, 0x17, 0x0e, 0xcb, 0x56, 0x41, 0xf8, 0xe3, 0x48, 0xa9, 0xc4, 0x15, 0xec, 0xb8, 0xc6, 0xce, 0x1e, 0xd8, 0x2a, 0xa8, 0x96, 0xd4, 0x51, 0x26, 0x3b, 0x9a, 0x1e, 0x66, 0x5a, - 0x60, 0x71, 0xe3, 0xca, 0x17, 0xe0, 0xc2, 0x81, 0x0f, 0x01, 0x55, 0x7c, 0x84, 0xad, 0xe2, 0xb2, - 0x37, 0x6e, 0x14, 0x84, 0x2f, 0x42, 0xf5, 0xdf, 0xe9, 0x1e, 0x29, 0xce, 0x1e, 0xf6, 0xd6, 0xef, - 0x6f, 0xbf, 0x7e, 0xfd, 0xeb, 0xf7, 0xde, 0x0c, 0xb4, 0x8b, 0xd7, 0x38, 0x9f, 0x1c, 0x65, 0x39, - 0x65, 0x14, 0xd5, 0x05, 0x71, 0xe7, 0xc1, 0x34, 0x66, 0xaf, 0xe7, 0xa3, 0xa3, 0x31, 0x9d, 0x1d, - 0x4f, 0xe9, 0x94, 0x1e, 0x0b, 0xe9, 0x68, 0xfe, 0x4a, 0x50, 0x82, 0x10, 0x2b, 0x69, 0x75, 0x67, - 0x9b, 0xc5, 0x33, 0x52, 0x30, 0x3c, 0xcb, 0x14, 0x63, 0x2b, 0x8b, 0x33, 0x92, 0xc4, 0x29, 0x51, - 0x34, 0x64, 0x09, 0x4e, 0xe5, 0x3a, 0xfc, 0x97, 0x07, 0x5b, 0x97, 0x7c, 0x97, 0xe2, 0x9c, 0x30, - 0x3c, 0xc1, 0x0c, 0xa3, 0x8f, 0xa0, 0x71, 0x41, 0x93, 0x78, 0xbc, 0xe8, 0x7a, 0x3d, 0xef, 0x60, - 0xeb, 0x64, 0xf3, 0x48, 0xc6, 0x24, 0x99, 0x91, 0x12, 0xa2, 0x1e, 0xb4, 0xa5, 0x61, 0x9f, 0xce, - 0x53, 0xd6, 0xad, 0xf5, 0xbc, 0x83, 0xcd, 0xc8, 0x66, 0xa1, 0x2e, 0x6c, 0x7c, 0x46, 0xf2, 0x22, - 0xa6, 0x69, 0x77, 0x5d, 0x48, 0x35, 0x89, 0xee, 0x42, 0xeb, 0x74, 0x3c, 0xe6, 0x4a, 0x67, 0x83, - 0xae, 0xdf, 0xf3, 0x0e, 0xfc, 0xa8, 0x64, 0xa0, 0x03, 0xd8, 0x3e, 0xc7, 0xd7, 0x11, 0xc9, 0x92, - 0x78, 0x8c, 0xa5, 0xf7, 0xba, 0xb0, 0xaf, 0xb2, 0xd1, 0x1d, 0x68, 0x8a, 0x0d, 0xcf, 0x06, 0x45, - 0xb7, 0xd1, 0x5b, 0x3f, 0xf0, 0x23, 0x43, 0x87, 0xff, 0xf0, 0x00, 0xae, 0xf0, 0x28, 0x21, 0x82, - 0xc3, 0x83, 0x11, 0xd4, 0xd9, 0x40, 0x1c, 0xcb, 0x8f, 0x34, 0xc9, 0x25, 0xca, 0x48, 0x1c, 0xc2, - 0x8f, 0x34, 0x69, 0x65, 0x62, 0xfd, 0xa6, 0x4c, 0x58, 0xe7, 0xf4, 0xdd, 0x73, 0x7e, 0x02, 0x4d, - 0x15, 0x6f, 0xd1, 0xad, 0xf7, 0xd6, 0x0f, 0xda, 0x27, 0xb7, 0x94, 0x0b, 0xb1, 0x85, 0x92, 0x3d, - 0xf6, 0xbf, 0xfc, 0xf7, 0xb7, 0xd7, 0x22, 0xa3, 0x1a, 0xfe, 0xd1, 0x83, 0x8e, 0xad, 0xc0, 0xf3, - 0xa5, 0x96, 0x26, 0xfc, 0x92, 0x81, 0xbe, 0x07, 0xf5, 0x4b, 0x86, 0x19, 0x11, 0xe1, 0x6f, 0x99, - 0x2d, 0x94, 0x82, 0x10, 0x45, 0x52, 0x03, 0x6d, 0x41, 0xad, 0x3f, 0x14, 0xa7, 0x69, 0x45, 0xb5, - 0xfe, 0xb0, 0x1a, 0xba, 0x6f, 0x42, 0x0f, 0xff, 0xec, 0xc3, 0x46, 0x44, 0x7e, 0x3b, 0x27, 0x05, - 0x93, 0xdb, 0x8b, 0xa5, 0xbd, 0xbd, 0x62, 0xa0, 0xfb, 0xd0, 0x8a, 0x2e, 0xfa, 0xe7, 0x84, 0xbd, - 0xa6, 0x13, 0x15, 0x82, 0x4e, 0x94, 0x64, 0x46, 0xa5, 0x1c, 0x0d, 0xa0, 0xd3, 0xcf, 0x09, 0x66, - 0xf2, 0x56, 0x0a, 0x11, 0x4a, 0xfb, 0xe4, 0x8e, 0xd2, 0xb7, 0x45, 0x6a, 0x03, 0x95, 0x1c, 0xc7, - 0x8a, 0x7b, 0x19, 0x90, 0x84, 0x18, 0x2f, 0xbe, 0xe3, 0xc5, 0x16, 0x55, 0xbc, 0xd8, 0x22, 0xf4, - 0x13, 0x68, 0x3d, 0x23, 0x38, 0x67, 0x23, 0x82, 0x25, 0xc2, 0xda, 0x27, 0x1f, 0x28, 0x17, 0x86, - 0xef, 0xda, 0x97, 0xfa, 0xdc, 0xf8, 0x29, 0x61, 0x6a, 0xff, 0x86, 0x63, 0x6c, 0xf8, 0x15, 0x63, - 0xc3, 0xe7, 0xc6, 0xea, 0x7e, 0xf1, 0xa4, 0xbb, 0xe1, 0x18, 0x1b, 0x7e, 0xc5, 0xd8, 0xf0, 0xd1, - 0x27, 0xb0, 0x71, 0x81, 0xe7, 0x05, 0xe9, 0x0f, 0xbb, 0x4d, 0x61, 0x7a, 0x5b, 0xc3, 0x52, 0x72, - 0x5d, 0x43, 0xad, 0x8b, 0x9e, 0xc2, 0xe6, 0x53, 0xc2, 0xb8, 0x58, 0x01, 0xb2, 0x25, 0x8c, 0xbf, - 0x55, 0x06, 0x5d, 0xca, 0x5c, 0x17, 0xae, 0x5d, 0xf8, 0x77, 0x9f, 0xa3, 0xba, 0xc8, 0x68, 0x5a, - 0x90, 0x6f, 0x12, 0x1a, 0xbb, 0x50, 0x7f, 0x92, 0xe7, 0x34, 0x17, 0x98, 0xe8, 0x44, 0x92, 0x40, - 0x4f, 0x2a, 0x80, 0xf1, 0x9d, 0xa8, 0x5d, 0xc0, 0xc8, 0x98, 0x56, 0x22, 0xe6, 0x49, 0x05, 0x31, - 0x75, 0xc7, 0x8d, 0x8b, 0x18, 0xd7, 0x8d, 0x03, 0x99, 0x47, 0x36, 0x64, 0xe4, 0xad, 0x77, 0x97, - 0x21, 0xe3, 0x38, 0xb0, 0x30, 0xf3, 0xc8, 0xc6, 0xcc, 0x86, 0x63, 0x6d, 0x61, 0xc6, 0xb5, 0x2e, - 0x41, 0xf3, 0xc8, 0x06, 0x4d, 0xd3, 0xb1, 0xb6, 0x40, 0xe3, 0x5a, 0x97, 0xa8, 0xf9, 0x51, 0x89, - 0x1a, 0x79, 0xf1, 0x7b, 0x55, 0xd4, 0x38, 0x96, 0x06, 0x36, 0xcf, 0xaa, 0xb0, 0x01, 0x61, 0x7d, - 0x77, 0x35, 0x6c, 0x1c, 0x1f, 0x55, 0xdc, 0x78, 0xd0, 0x7c, 0x91, 0x91, 0x1c, 0x33, 0x9a, 0xa3, - 0xef, 0x80, 0x7f, 0xb5, 0xc8, 0x48, 0xa5, 0xc5, 0xbc, 0xc8, 0x38, 0x33, 0x12, 0x22, 0xf4, 0xa9, - 0x5d, 0xbf, 0x05, 0x7a, 0xda, 0x27, 0x3b, 0x4a, 0xb1, 0x14, 0xa8, 0xbd, 0xec, 0x52, 0xff, 0x90, - 0x57, 0x2e, 0xb1, 0xa9, 0x2a, 0x2f, 0x37, 0x14, 0x5d, 0xad, 0x69, 0xf7, 0x07, 0xdf, 0xe9, 0x0f, - 0xe1, 0xaf, 0xe1, 0xd6, 0x8a, 0xba, 0xc4, 0x4b, 0xa9, 0x81, 0x7c, 0xed, 0x6c, 0x80, 0x3e, 0x85, - 0xa6, 0x6e, 0xa1, 0x2a, 0xd8, 0xdb, 0xf6, 0xb6, 0xa6, 0xbf, 0xea, 0x6a, 0xaf, 0xe9, 0x70, 0x0f, - 0x76, 0x57, 0xc1, 0x38, 0xfc, 0x08, 0x6e, 0xad, 0xa8, 0x64, 0xd5, 0x7d, 0xb9, 0xf9, 0x2a, 0xf8, - 0x86, 0x97, 0x10, 0x54, 0xab, 0x98, 0x2a, 0xff, 0x9e, 0x29, 0xff, 0xc7, 0xd0, 0x50, 0x68, 0xac, - 0x89, 0xee, 0xf4, 0xce, 0xf4, 0x2a, 0xb5, 0xf0, 0x19, 0xec, 0x2c, 0xe1, 0x1c, 0x3d, 0x84, 0x96, - 0xbe, 0xd7, 0xa2, 0xeb, 0x09, 0x47, 0xdb, 0xe6, 0x42, 0x25, 0x5f, 0xe3, 0xd1, 0xe8, 0x85, 0x9f, - 0x43, 0x50, 0xad, 0x93, 0xdf, 0x5c, 0x4a, 0x07, 0xb0, 0xb3, 0xf4, 0xa0, 0xac, 0xc3, 0x7a, 0x5f, - 0xef, 0xb0, 0xff, 0xf4, 0x20, 0xa8, 0x96, 0x63, 0xf4, 0x00, 0xea, 0x12, 0x90, 0xde, 0xcd, 0x80, - 0x94, 0x5a, 0x2a, 0xe3, 0x35, 0x93, 0xf1, 0x3d, 0x68, 0xa8, 0x72, 0x28, 0x47, 0x22, 0x45, 0xa1, - 0xef, 0x43, 0xfd, 0x02, 0xe7, 0x78, 0xa6, 0xea, 0x5b, 0x60, 0x7a, 0x38, 0x9e, 0x08, 0xbe, 0xf6, - 0x2a, 0x08, 0x74, 0x02, 0x0d, 0x2e, 0x39, 0xd5, 0x6d, 0x6b, 0xf7, 0xa8, 0x9c, 0xf9, 0xae, 0xf4, - 0x4a, 0x9f, 0x46, 0x6a, 0x86, 0x0f, 0x60, 0x67, 0xa9, 0x4c, 0x70, 0xd4, 0x5f, 0xe0, 0x45, 0x42, - 0xb1, 0x3c, 0x4f, 0x27, 0xd2, 0x64, 0xd8, 0x83, 0x2d, 0xb7, 0x9f, 0x58, 0xb7, 0xd3, 0x12, 0xc0, - 0xdb, 0x81, 0xed, 0x4a, 0xed, 0x08, 0x3f, 0x86, 0xdd, 0x55, 0x7d, 0x64, 0xc9, 0x74, 0x08, 0xb7, - 0x57, 0x16, 0x0e, 0x67, 0x60, 0xf2, 0xbe, 0xfe, 0xc0, 0xf4, 0xb7, 0x0d, 0xde, 0x86, 0x54, 0xaa, - 0x44, 0x83, 0xcc, 0xe9, 0x98, 0x14, 0x85, 0xba, 0xa4, 0xdb, 0x47, 0x66, 0x02, 0x56, 0x82, 0xb3, - 0xf4, 0x15, 0x35, 0x95, 0x4e, 0xb2, 0xd0, 0x0f, 0xa0, 0x79, 0x75, 0x9d, 0x8a, 0x8b, 0x54, 0x68, - 0xd3, 0x28, 0xd6, 0x6c, 0xbd, 0xaf, 0xa6, 0x79, 0x89, 0xe2, 0x73, 0x55, 0x21, 0xaf, 0x6e, 0xdd, - 0x41, 0x44, 0x29, 0xd0, 0x25, 0xaa, 0xe4, 0xa0, 0x1f, 0x42, 0xeb, 0x32, 0xfe, 0x03, 0x59, 0x75, - 0xe5, 0x86, 0x6f, 0x6a, 0xb8, 0x66, 0xa0, 0x1f, 0x43, 0x3b, 0xc2, 0xe9, 0x94, 0xa8, 0xfd, 0xe4, - 0xdd, 0x23, 0x0d, 0x95, 0x52, 0xa2, 0x2c, 0x6d, 0x65, 0x94, 0xc1, 0x87, 0x3c, 0xe5, 0x34, 0x99, - 0xcf, 0xf4, 0x33, 0xb9, 0x1c, 0xe3, 0x94, 0x27, 0x42, 0x7a, 0x93, 0xdd, 0xec, 0x9e, 0x55, 0xd7, - 0xdf, 0xa9, 0xab, 0xfc, 0xdf, 0xec, 0x10, 0xfd, 0x12, 0x76, 0xce, 0x49, 0x3e, 0x25, 0x2f, 0x46, - 0x6f, 0xc8, 0x58, 0xe7, 0xc8, 0xed, 0x7a, 0x4b, 0x72, 0xe5, 0x79, 0xd9, 0x90, 0xdf, 0xce, 0x73, - 0xb2, 0x90, 0x4e, 0x9a, 0xce, 0xed, 0x68, 0xb6, 0xbe, 0x1d, 0x4d, 0xa3, 0x33, 0x08, 0x38, 0x28, - 0x48, 0xfe, 0x78, 0x1e, 0x27, 0x12, 0x1b, 0xaa, 0xf7, 0x7d, 0x60, 0x3d, 0x2f, 0x5b, 0xac, 0x5c, - 0x2c, 0x99, 0xf1, 0xec, 0x5d, 0xe4, 0xf1, 0x0c, 0xe7, 0x8b, 0xe7, 0x64, 0x51, 0x9c, 0xe3, 0xc5, - 0x63, 0x72, 0x4e, 0x27, 0xf1, 0xab, 0x98, 0x28, 0xbf, 0xe0, 0x64, 0xef, 0x46, 0x5d, 0x9d, 0xbd, - 0x1b, 0x95, 0xd0, 0x2f, 0x60, 0x93, 0x47, 0x31, 0x24, 0xd7, 0x4c, 0xee, 0xd0, 0x56, 0x2f, 0xbd, - 0x8c, 0xdc, 0xc8, 0x74, 0xbf, 0x75, 0x98, 0xa8, 0x0f, 0x5b, 0x9c, 0xd1, 0x4f, 0x68, 0xa1, 0x80, - 0xd6, 0x71, 0x6a, 0xa8, 0x2b, 0x54, 0x3e, 0x2a, 0x26, 0xe8, 0x73, 0xd8, 0xeb, 0xd3, 0x24, 0x21, - 0x63, 0x76, 0x45, 0x67, 0xa3, 0x82, 0xd1, 0x54, 0xa3, 0x6f, 0x53, 0x38, 0xfb, 0x50, 0x0f, 0x62, - 0x2b, 0x95, 0x94, 0xd3, 0x77, 0xb8, 0x08, 0xff, 0xe4, 0x95, 0x4f, 0xce, 0xfd, 0x26, 0xf4, 0xaa, - 0xdf, 0x84, 0xfb, 0x00, 0x03, 0xcc, 0xf0, 0x08, 0x17, 0xc4, 0x7c, 0xa7, 0x59, 0x1c, 0x14, 0x42, - 0x47, 0x53, 0x43, 0x3c, 0x23, 0xea, 0x13, 0xc7, 0xe1, 0xf1, 0x1d, 0xc4, 0x56, 0x42, 0xc1, 0x17, - 0x0a, 0x25, 0x23, 0xec, 0xd9, 0x6f, 0x19, 0x21, 0xf0, 0x8b, 0x45, 0x3a, 0x16, 0x81, 0x34, 0x23, - 0xb1, 0x0e, 0xef, 0x5b, 0x8f, 0x96, 0x07, 0x34, 0xe6, 0xd8, 0x4f, 0x85, 0x37, 0x59, 0xda, 0x2c, - 0x4e, 0xf8, 0x1b, 0xe7, 0xad, 0xa2, 0x1e, 0xd4, 0x9f, 0x5c, 0x67, 0xa6, 0x3f, 0xc2, 0x91, 0xf8, - 0x06, 0xe7, 0xac, 0x48, 0x0a, 0xd0, 0x11, 0xa0, 0x97, 0xe9, 0x98, 0xce, 0x66, 0x31, 0x63, 0x64, - 0xa2, 0xb0, 0x2f, 0x4e, 0xda, 0x89, 0x56, 0x48, 0xc2, 0x9f, 0xbd, 0xeb, 0x66, 0xd0, 0x3d, 0xd8, - 0x1c, 0x4b, 0x89, 0xf5, 0x1d, 0xef, 0x47, 0x2e, 0x33, 0xfc, 0xf9, 0x7b, 0x0a, 0xc2, 0x7b, 0x4f, - 0x78, 0xb7, 0x7c, 0x91, 0x28, 0x80, 0xf5, 0xe7, 0x64, 0xa1, 0x7a, 0x08, 0x5f, 0x86, 0x7f, 0xf1, - 0x96, 0x5f, 0x1f, 0x6f, 0x37, 0x39, 0x49, 0xf8, 0xa5, 0xe8, 0x76, 0xa3, 0x48, 0xb4, 0x0f, 0x3e, - 0xb9, 0xce, 0x72, 0x55, 0x78, 0xed, 0xf4, 0x08, 0x3e, 0xff, 0xd2, 0x2f, 0xc6, 0x38, 0x15, 0x33, - 0x23, 0xbf, 0xdb, 0x7a, 0x64, 0x68, 0x74, 0x02, 0xbb, 0x4c, 0xa7, 0xe0, 0x34, 0xcb, 0x92, 0x85, - 0x3a, 0xb6, 0x2f, 0xf4, 0x56, 0xca, 0xc2, 0xef, 0xc2, 0x8e, 0x15, 0x5d, 0x44, 0x8a, 0x79, 0xc2, - 0xf8, 0xa5, 0xcf, 0xe7, 0xb1, 0x6e, 0x85, 0x62, 0x1d, 0xfe, 0xb4, 0xf2, 0x0e, 0x57, 0x29, 0xf1, - 0x73, 0xc9, 0xc4, 0xc8, 0x41, 0xaa, 0x15, 0x69, 0x32, 0x3c, 0x90, 0x8f, 0x90, 0x9b, 0xab, 0x4d, - 0xf6, 0xa0, 0x91, 0x8b, 0x95, 0xf2, 0xa0, 0xa8, 0xf0, 0x5e, 0xf5, 0xb9, 0xae, 0x0c, 0x07, 0xbf, - 0xa7, 0x10, 0x71, 0xa3, 0x57, 0x39, 0x9d, 0x69, 0x23, 0xbe, 0xe6, 0xed, 0x97, 0x51, 0x05, 0xa5, - 0x1a, 0xa3, 0xfc, 0x21, 0x7c, 0x41, 0x16, 0x9f, 0x91, 0x31, 0x33, 0x5f, 0x5b, 0x25, 0x23, 0x7c, - 0xb9, 0xa2, 0x6e, 0xf3, 0xfc, 0xd3, 0xd1, 0x9b, 0x82, 0x3f, 0x10, 0x01, 0xe1, 0x4e, 0x64, 0x68, - 0x8e, 0x37, 0x86, 0xf3, 0x29, 0x61, 0x2f, 0x46, 0x6f, 0xf8, 0x03, 0x51, 0xa3, 0x8d, 0xcb, 0x3c, - 0xbc, 0xaf, 0x7f, 0xa6, 0xa0, 0x26, 0xf8, 0x43, 0x9a, 0x92, 0x60, 0x0d, 0x6d, 0x42, 0xeb, 0x02, - 0xe7, 0x2c, 0x66, 0x31, 0x4d, 0x03, 0x8f, 0x0b, 0x9e, 0xe1, 0xe2, 0x75, 0x50, 0x3b, 0x7c, 0x0e, - 0x0d, 0xf9, 0x2d, 0x80, 0xb6, 0x00, 0x4e, 0x27, 0xba, 0xf1, 0x07, 0x6b, 0x68, 0x07, 0x36, 0xe5, - 0xb8, 0xab, 0x59, 0x1e, 0xf7, 0x22, 0x59, 0xa7, 0x49, 0x12, 0xd4, 0xd0, 0x36, 0xb4, 0xe5, 0x3c, - 0x2d, 0xde, 0x76, 0xb0, 0x7e, 0xf8, 0x31, 0x6c, 0xf4, 0x87, 0xf2, 0xff, 0x47, 0x03, 0x6a, 0x2f, - 0xb3, 0x60, 0x0d, 0xb5, 0xf8, 0xb8, 0x35, 0x2f, 0x88, 0xdc, 0x74, 0x40, 0x7f, 0x9f, 0x06, 0xb5, - 0xc3, 0x97, 0xd0, 0xb1, 0xff, 0x99, 0x88, 0xad, 0x93, 0x84, 0x8e, 0x31, 0x8b, 0xd3, 0xa9, 0x8c, - 0x56, 0xd1, 0x64, 0x12, 0x78, 0xa8, 0x0d, 0x1b, 0xd1, 0x3c, 0x4d, 0xb9, 0xac, 0x86, 0x00, 0x1a, - 0xe7, 0xf4, 0x77, 0x7c, 0xbd, 0xce, 0xf5, 0xcc, 0x93, 0x0c, 0xfc, 0xc3, 0x85, 0x1e, 0xf9, 0xb8, - 0xc0, 0x4c, 0xcf, 0xc1, 0x1a, 0x0a, 0xdc, 0x4f, 0xdb, 0xc0, 0xe3, 0x1c, 0x7b, 0x96, 0x0f, 0x6a, - 0xdc, 0xc4, 0x4c, 0xb2, 0xd2, 0xb5, 0x19, 0xe2, 0x02, 0x9f, 0x87, 0xa0, 0x46, 0xb0, 0xa0, 0xce, - 0x33, 0xe3, 0x0c, 0x55, 0x41, 0xe3, 0xf1, 0xe0, 0xab, 0xff, 0xee, 0x7b, 0x5f, 0xbe, 0xdd, 0xf7, - 0xbe, 0x7a, 0xbb, 0xef, 0xfd, 0xe7, 0xed, 0xfe, 0xda, 0x5f, 0xff, 0xb7, 0xef, 0xfd, 0xea, 0xc8, - 0xfa, 0x9f, 0x38, 0xc3, 0x2c, 0x8f, 0xaf, 0x69, 0x1e, 0x4f, 0xe3, 0x54, 0x13, 0x29, 0x39, 0xce, - 0xbe, 0x98, 0x1e, 0x67, 0xa3, 0x63, 0x51, 0xdb, 0x47, 0x0d, 0xf1, 0xab, 0xf0, 0xe1, 0xff, 0x03, - 0x00, 0x00, 0xff, 0xff, 0x47, 0xd8, 0xa1, 0xa8, 0x9c, 0x14, 0x00, 0x00, + 0x60, 0x71, 0x82, 0x2b, 0x5f, 0x80, 0x2b, 0x1f, 0x02, 0xaa, 0xf8, 0x08, 0x5b, 0xc5, 0x65, 0x6f, + 0xdc, 0x28, 0x08, 0x5f, 0x84, 0xea, 0xbf, 0xd3, 0x3d, 0x92, 0x9d, 0x3d, 0xec, 0xad, 0xdf, 0xff, + 0xd7, 0xdd, 0xbf, 0x7e, 0xef, 0xcd, 0x40, 0xbb, 0x78, 0x83, 0xf3, 0xf1, 0x61, 0x96, 0x53, 0x46, + 0x51, 0x5d, 0x10, 0xbb, 0x0f, 0x27, 0x31, 0x7b, 0x33, 0x1b, 0x1e, 0x8e, 0xe8, 0xf4, 0x68, 0x42, + 0x27, 0xf4, 0x48, 0x48, 0x87, 0xb3, 0xd7, 0x82, 0x12, 0x84, 0x58, 0x49, 0xab, 0xdd, 0x4d, 0x16, + 0x4f, 0x49, 0xc1, 0xf0, 0x34, 0x53, 0x8c, 0x8d, 0x2c, 0xce, 0x48, 0x12, 0xa7, 0x44, 0xd1, 0x90, + 0x25, 0x38, 0x95, 0xeb, 0xf0, 0x5f, 0x1e, 0x6c, 0x5c, 0xf2, 0x28, 0xc5, 0x19, 0x61, 0x78, 0x8c, + 0x19, 0x46, 0x1f, 0x41, 0xe3, 0x82, 0x26, 0xf1, 0x68, 0xde, 0xf5, 0x7a, 0xde, 0xfe, 0xc6, 0xf1, + 0xfa, 0xa1, 0xcc, 0x49, 0x32, 0x23, 0x25, 0x44, 0x3d, 0x68, 0x4b, 0xc3, 0x3e, 0x9d, 0xa5, 0xac, + 0x5b, 0xeb, 0x79, 0xfb, 0xeb, 0x91, 0xcd, 0x42, 0x5d, 0x58, 0xfb, 0x8c, 0xe4, 0x45, 0x4c, 0xd3, + 0xee, 0xaa, 0x90, 0x6a, 0x12, 0xdd, 0x83, 0xd6, 0xc9, 0x68, 0xc4, 0x95, 0x4e, 0x07, 0x5d, 0xbf, + 0xe7, 0xed, 0xfb, 0x51, 0xc9, 0x40, 0xfb, 0xb0, 0x79, 0x86, 0xaf, 0x23, 0x92, 0x25, 0xf1, 0x08, + 0x4b, 0xef, 0x75, 0x61, 0x5f, 0x65, 0xa3, 0x5d, 0x68, 0x8a, 0x80, 0xa7, 0x83, 0xa2, 0xdb, 0xe8, + 0xad, 0xee, 0xfb, 0x91, 0xa1, 0xc3, 0x7f, 0x78, 0x00, 0x57, 0x78, 0x98, 0x10, 0xc1, 0xe1, 0xc9, + 0x08, 0xea, 0x74, 0x20, 0xb6, 0xe5, 0x47, 0x9a, 0xe4, 0x12, 0x65, 0x24, 0x36, 0xe1, 0x47, 0x9a, + 0xb4, 0x4e, 0x62, 0xf5, 0xb6, 0x93, 0xb0, 0xf6, 0xe9, 0xbb, 0xfb, 0xfc, 0x04, 0x9a, 0x2a, 0xdf, + 0xa2, 0x5b, 0xef, 0xad, 0xee, 0xb7, 0x8f, 0xef, 0x28, 0x17, 0x22, 0x84, 0x92, 0x3d, 0xf1, 0xbf, + 0xfc, 0xf7, 0xb7, 0x57, 0x22, 0xa3, 0x1a, 0xfe, 0xc9, 0x83, 0x8e, 0xad, 0xc0, 0xcf, 0x4b, 0x2d, + 0x4d, 0xfa, 0x25, 0x03, 0x7d, 0x0f, 0xea, 0x97, 0x0c, 0x33, 0x22, 0xd2, 0xdf, 0x30, 0x21, 0x94, + 0x82, 0x10, 0x45, 0x52, 0x03, 0x6d, 0x40, 0xad, 0x7f, 0x2e, 0x76, 0xd3, 0x8a, 0x6a, 0xfd, 0xf3, + 0x6a, 0xea, 0xbe, 0x49, 0x3d, 0xfc, 0x8b, 0x0f, 0x6b, 0x11, 0xf9, 0xed, 0x8c, 0x14, 0x4c, 0x86, + 0x17, 0x4b, 0x3b, 0xbc, 0x62, 0xa0, 0x07, 0xd0, 0x8a, 0x2e, 0xfa, 0x67, 0x84, 0xbd, 0xa1, 0x63, + 0x95, 0x82, 0x3e, 0x28, 0xc9, 0x8c, 0x4a, 0x39, 0x1a, 0x40, 0xa7, 0x9f, 0x13, 0xcc, 0xe4, 0xad, + 0x14, 0x22, 0x95, 0xf6, 0xf1, 0xae, 0xd2, 0xb7, 0x45, 0x2a, 0x80, 0x3a, 0x1c, 0xc7, 0x8a, 0x7b, + 0x19, 0x90, 0x84, 0x18, 0x2f, 0xbe, 0xe3, 0xc5, 0x16, 0x55, 0xbc, 0xd8, 0x22, 0xf4, 0x13, 0x68, + 0x3d, 0x27, 0x38, 0x67, 0x43, 0x82, 0x25, 0xc2, 0xda, 0xc7, 0x1f, 0x28, 0x17, 0x86, 0xef, 0xda, + 0x97, 0xfa, 0xdc, 0xf8, 0x19, 0x61, 0x2a, 0x7e, 0xc3, 0x31, 0x36, 0xfc, 0x8a, 0xb1, 0xe1, 0x73, + 0x63, 0x75, 0xbf, 0x78, 0xdc, 0x5d, 0x73, 0x8c, 0x0d, 0xbf, 0x62, 0x6c, 0xf8, 0xe8, 0x13, 0x58, + 0xbb, 0xc0, 0xb3, 0x82, 0xf4, 0xcf, 0xbb, 0x4d, 0x61, 0x7a, 0x57, 0xc3, 0x52, 0x72, 0x5d, 0x43, + 0xad, 0x8b, 0x9e, 0xc1, 0xfa, 0x33, 0xc2, 0xb8, 0x58, 0x01, 0xb2, 0x25, 0x8c, 0xbf, 0x55, 0x26, + 0x5d, 0xca, 0x5c, 0x17, 0xae, 0x5d, 0xf8, 0x77, 0x9f, 0xa3, 0xba, 0xc8, 0x68, 0x5a, 0x90, 0x6f, + 0x12, 0x1a, 0xdb, 0x50, 0x7f, 0x9a, 0xe7, 0x34, 0x17, 0x98, 0xe8, 0x44, 0x92, 0x40, 0x4f, 0x2b, + 0x80, 0xf1, 0x9d, 0xac, 0x5d, 0xc0, 0xc8, 0x9c, 0x96, 0x22, 0xe6, 0x69, 0x05, 0x31, 0x75, 0xc7, + 0x8d, 0x8b, 0x18, 0xd7, 0x8d, 0x03, 0x99, 0xc7, 0x36, 0x64, 0xe4, 0xad, 0x77, 0x17, 0x21, 0xe3, + 0x38, 0xb0, 0x30, 0xf3, 0xd8, 0xc6, 0xcc, 0x9a, 0x63, 0x6d, 0x61, 0xc6, 0xb5, 0x2e, 0x41, 0xf3, + 0xd8, 0x06, 0x4d, 0xd3, 0xb1, 0xb6, 0x40, 0xe3, 0x5a, 0x97, 0xa8, 0xf9, 0x51, 0x89, 0x1a, 0x79, + 0xf1, 0x3b, 0x55, 0xd4, 0x38, 0x96, 0x06, 0x36, 0xcf, 0xab, 0xb0, 0x01, 0x61, 0x7d, 0x6f, 0x39, + 0x6c, 0x1c, 0x1f, 0x55, 0xdc, 0x78, 0xd0, 0x7c, 0x99, 0x91, 0x1c, 0x33, 0x9a, 0xa3, 0xef, 0x80, + 0x7f, 0x35, 0xcf, 0x48, 0xa5, 0xc5, 0xbc, 0xcc, 0x38, 0x33, 0x12, 0x22, 0xf4, 0xa9, 0x5d, 0xbf, + 0x05, 0x7a, 0xda, 0xc7, 0x5b, 0x4a, 0xb1, 0x14, 0xa8, 0x58, 0x76, 0xa9, 0x7f, 0xc4, 0x2b, 0x97, + 0x08, 0xaa, 0xca, 0xcb, 0x2d, 0x45, 0x57, 0x6b, 0xda, 0xfd, 0xc1, 0x77, 0xfa, 0x43, 0xf8, 0x6b, + 0xb8, 0xb3, 0xa4, 0x2e, 0xf1, 0x52, 0x6a, 0x20, 0x5f, 0x3b, 0x1d, 0xa0, 0x4f, 0xa1, 0xa9, 0x5b, + 0xa8, 0x4a, 0xf6, 0xae, 0x1d, 0xd6, 0xf4, 0x57, 0x5d, 0xed, 0x35, 0x1d, 0xee, 0xc0, 0xf6, 0x32, + 0x18, 0x87, 0x1f, 0xc1, 0x9d, 0x25, 0x95, 0xac, 0x1a, 0x97, 0x9b, 0x2f, 0x83, 0x6f, 0x78, 0x09, + 0x41, 0xb5, 0x8a, 0xa9, 0xf2, 0xef, 0x99, 0xf2, 0x7f, 0x04, 0x0d, 0x85, 0xc6, 0x9a, 0xe8, 0x4e, + 0x37, 0x1e, 0xaf, 0x52, 0x0b, 0x9f, 0xc3, 0xd6, 0x02, 0xce, 0xd1, 0x23, 0x68, 0xe9, 0x7b, 0x2d, + 0xba, 0x9e, 0x70, 0xb4, 0x69, 0x2e, 0x54, 0xf2, 0x35, 0x1e, 0x8d, 0x5e, 0xf8, 0x39, 0x04, 0xd5, + 0x3a, 0xf9, 0xcd, 0x1d, 0xe9, 0x00, 0xb6, 0x16, 0x1e, 0x94, 0xb5, 0x59, 0xef, 0xeb, 0x6d, 0xf6, + 0x9f, 0x1e, 0x04, 0xd5, 0x72, 0x8c, 0x1e, 0x42, 0x5d, 0x02, 0xd2, 0xbb, 0x1d, 0x90, 0x52, 0x4b, + 0x9d, 0x78, 0xcd, 0x9c, 0xf8, 0x0e, 0x34, 0x54, 0x39, 0x94, 0x23, 0x91, 0xa2, 0xd0, 0xf7, 0xa1, + 0x7e, 0x81, 0x73, 0x3c, 0x55, 0xf5, 0x2d, 0x30, 0x3d, 0x1c, 0x8f, 0x05, 0x5f, 0x7b, 0x15, 0x04, + 0x3a, 0x86, 0x06, 0x97, 0x9c, 0xe8, 0xb6, 0xb5, 0x7d, 0x58, 0xce, 0x7c, 0x57, 0x7a, 0xa5, 0x77, + 0x23, 0x35, 0xc3, 0x87, 0xb0, 0xb5, 0x50, 0x26, 0x38, 0xea, 0x2f, 0xf0, 0x3c, 0xa1, 0x58, 0xee, + 0xa7, 0x13, 0x69, 0x32, 0xec, 0xc1, 0x86, 0xdb, 0x4f, 0xac, 0xdb, 0x69, 0x09, 0xe0, 0x6d, 0xc1, + 0x66, 0xa5, 0x76, 0x84, 0x1f, 0xc3, 0xf6, 0xb2, 0x3e, 0xb2, 0x60, 0x7a, 0x0e, 0x77, 0x97, 0x16, + 0x0e, 0x67, 0x60, 0xf2, 0xbe, 0xfe, 0xc0, 0xf4, 0xb7, 0x35, 0xde, 0x86, 0xd4, 0x51, 0x89, 0x06, + 0x99, 0xd3, 0x11, 0x29, 0x0a, 0x75, 0x49, 0x77, 0x0f, 0xcd, 0x04, 0xac, 0x04, 0xa7, 0xe9, 0x6b, + 0x6a, 0x2a, 0x9d, 0x64, 0xa1, 0x1f, 0x40, 0xf3, 0xea, 0x3a, 0x15, 0x17, 0xa9, 0xd0, 0xa6, 0x51, + 0xac, 0xd9, 0x3a, 0xae, 0xa6, 0x79, 0x89, 0xe2, 0x73, 0x55, 0x21, 0xaf, 0x6e, 0xd5, 0x41, 0x44, + 0x29, 0xd0, 0x25, 0xaa, 0xe4, 0xa0, 0x1f, 0x42, 0xeb, 0x32, 0xfe, 0x03, 0x59, 0x76, 0xe5, 0x86, + 0x6f, 0x6a, 0xb8, 0x66, 0xa0, 0x1f, 0x43, 0x3b, 0xc2, 0xe9, 0x84, 0xa8, 0x78, 0xf2, 0xee, 0x91, + 0x86, 0x4a, 0x29, 0x51, 0x96, 0xb6, 0x32, 0xca, 0xe0, 0x43, 0x7e, 0xe4, 0x34, 0x99, 0x4d, 0xf5, + 0x33, 0xb9, 0x1c, 0xe1, 0x94, 0x1f, 0x84, 0xf4, 0x26, 0xbb, 0xd9, 0x7d, 0xab, 0xae, 0xdf, 0xa8, + 0xab, 0xfc, 0xdf, 0xee, 0x10, 0xfd, 0x12, 0xb6, 0xce, 0x48, 0x3e, 0x21, 0x2f, 0x87, 0x6f, 0xc9, + 0x48, 0x9f, 0x91, 0xdb, 0xf5, 0x16, 0xe4, 0xca, 0xf3, 0xa2, 0x21, 0xbf, 0x9d, 0x17, 0x64, 0x2e, + 0x9d, 0x34, 0x9d, 0xdb, 0xd1, 0x6c, 0x7d, 0x3b, 0x9a, 0x46, 0xa7, 0x10, 0x70, 0x50, 0x90, 0xfc, + 0xc9, 0x2c, 0x4e, 0x24, 0x36, 0x54, 0xef, 0xfb, 0xc0, 0x7a, 0x5e, 0xb6, 0x58, 0xb9, 0x58, 0x30, + 0xe3, 0xa7, 0x77, 0x91, 0xc7, 0x53, 0x9c, 0xcf, 0x5f, 0x90, 0x79, 0x71, 0x86, 0xe7, 0x4f, 0xc8, + 0x19, 0x1d, 0xc7, 0xaf, 0x63, 0xa2, 0xfc, 0x82, 0x73, 0x7a, 0xb7, 0xea, 0xea, 0xd3, 0xbb, 0x55, + 0x09, 0xfd, 0x02, 0xd6, 0x79, 0x16, 0xe7, 0xe4, 0x9a, 0xc9, 0x08, 0x6d, 0xf5, 0xd2, 0xcb, 0xcc, + 0x8d, 0x4c, 0xf7, 0x5b, 0x87, 0x89, 0xfa, 0xb0, 0xc1, 0x19, 0xfd, 0x84, 0x16, 0x0a, 0x68, 0x1d, + 0xa7, 0x86, 0xba, 0x42, 0xe5, 0xa3, 0x62, 0x82, 0x3e, 0x87, 0x9d, 0x3e, 0x4d, 0x12, 0x32, 0x62, + 0x57, 0x74, 0x3a, 0x2c, 0x18, 0x4d, 0x35, 0xfa, 0xd6, 0x85, 0xb3, 0x0f, 0xf5, 0x20, 0xb6, 0x54, + 0x49, 0x39, 0xbd, 0xc1, 0x45, 0xf8, 0x67, 0xaf, 0x7c, 0x72, 0xee, 0x37, 0xa1, 0x57, 0xfd, 0x26, + 0xdc, 0x03, 0x18, 0x60, 0x86, 0x87, 0xb8, 0x20, 0xe6, 0x3b, 0xcd, 0xe2, 0xa0, 0x10, 0x3a, 0x9a, + 0x3a, 0xc7, 0x53, 0xa2, 0x3e, 0x71, 0x1c, 0x1e, 0x8f, 0x20, 0x42, 0x09, 0x05, 0x5f, 0x28, 0x94, + 0x8c, 0xb0, 0x67, 0xbf, 0x65, 0x84, 0xc0, 0x2f, 0xe6, 0xe9, 0x48, 0x24, 0xd2, 0x8c, 0xc4, 0x3a, + 0x7c, 0x60, 0x3d, 0x5a, 0x9e, 0xd0, 0x88, 0x63, 0x3f, 0x15, 0xde, 0x64, 0x69, 0xb3, 0x38, 0xe1, + 0x6f, 0x9c, 0xb7, 0x8a, 0x7a, 0x50, 0x7f, 0x7a, 0x9d, 0x99, 0xfe, 0x08, 0x87, 0xe2, 0x1b, 0x9c, + 0xb3, 0x22, 0x29, 0x40, 0x87, 0x80, 0x5e, 0xa5, 0x23, 0x3a, 0x9d, 0xc6, 0x8c, 0x91, 0xb1, 0xc2, + 0xbe, 0xd8, 0x69, 0x27, 0x5a, 0x22, 0x09, 0x7f, 0x76, 0xd3, 0xcd, 0xa0, 0xfb, 0xb0, 0x3e, 0x92, + 0x12, 0xeb, 0x3b, 0xde, 0x8f, 0x5c, 0x66, 0xf8, 0xf3, 0xf7, 0x14, 0x84, 0xf7, 0xee, 0xf0, 0x5e, + 0xf9, 0x22, 0x51, 0x00, 0xab, 0x2f, 0xc8, 0x5c, 0xf5, 0x10, 0xbe, 0x0c, 0xff, 0xe8, 0x2d, 0xbe, + 0x3e, 0xde, 0x6e, 0x72, 0x92, 0xf0, 0x4b, 0xd1, 0xed, 0x46, 0x91, 0x68, 0x0f, 0x7c, 0x72, 0x9d, + 0xe5, 0xaa, 0xf0, 0xda, 0xc7, 0x23, 0xf8, 0xe8, 0x18, 0xb6, 0x99, 0xde, 0xe6, 0x49, 0x96, 0x25, + 0x73, 0xeb, 0xc3, 0xbc, 0x1e, 0x2d, 0x95, 0x85, 0xdf, 0x85, 0x2d, 0x2b, 0x83, 0x88, 0x14, 0xb3, + 0x84, 0xf1, 0x8b, 0x9d, 0xcd, 0x62, 0xdd, 0xee, 0xc4, 0x3a, 0xfc, 0x69, 0xe5, 0xad, 0x2d, 0x53, + 0xe2, 0xb9, 0xcb, 0xcd, 0xcb, 0x61, 0xa9, 0x15, 0x69, 0x32, 0xdc, 0x97, 0x0f, 0x8d, 0x9b, 0xab, + 0x20, 0x3b, 0xd0, 0xc8, 0xc5, 0x4a, 0x79, 0x50, 0x54, 0x78, 0xbf, 0xfa, 0x24, 0x97, 0xa6, 0x83, + 0xdf, 0x53, 0x6c, 0xb8, 0xd1, 0xeb, 0x9c, 0x4e, 0xb5, 0x11, 0x5f, 0xf3, 0x16, 0xcb, 0xa8, 0x82, + 0x4b, 0x8d, 0x51, 0x0e, 0xf6, 0x2f, 0xc8, 0xfc, 0x33, 0x32, 0x62, 0xe6, 0x8b, 0xaa, 0x64, 0x84, + 0xaf, 0x96, 0xd4, 0x66, 0xb4, 0x0b, 0x4d, 0x3a, 0x7c, 0x5b, 0xf0, 0x47, 0x20, 0x60, 0xda, 0x89, + 0x0c, 0xcd, 0x31, 0xc5, 0x70, 0x3e, 0x21, 0xec, 0xe5, 0xf0, 0x2d, 0x7f, 0x04, 0x6a, 0x7c, 0x71, + 0x99, 0x07, 0x0f, 0xf4, 0x0f, 0x13, 0xd4, 0x04, 0xff, 0x9c, 0xa6, 0x24, 0x58, 0x41, 0xeb, 0xd0, + 0xba, 0xc0, 0x39, 0x8b, 0x59, 0x4c, 0xd3, 0xc0, 0xe3, 0x82, 0xe7, 0xb8, 0x78, 0x13, 0xd4, 0x0e, + 0x5e, 0x40, 0x43, 0xce, 0xfb, 0x68, 0x03, 0xe0, 0x64, 0xac, 0x9b, 0x7b, 0xb0, 0x82, 0xb6, 0x60, + 0x5d, 0x8e, 0xb4, 0x9a, 0xe5, 0x71, 0x2f, 0x92, 0x75, 0x92, 0x24, 0x41, 0x0d, 0x6d, 0x42, 0x5b, + 0xce, 0xcc, 0xe2, 0xfd, 0x06, 0xab, 0x07, 0x1f, 0xc3, 0x5a, 0xff, 0x5c, 0xfe, 0xe3, 0x68, 0x40, + 0xed, 0x55, 0x16, 0xac, 0xa0, 0x16, 0x1f, 0xa9, 0x66, 0x05, 0x91, 0x41, 0x07, 0xf4, 0xf7, 0x69, + 0x50, 0x3b, 0x78, 0x05, 0x1d, 0xfb, 0xbf, 0x88, 0x08, 0x9d, 0x24, 0x74, 0x84, 0x59, 0x9c, 0x4e, + 0x64, 0xb6, 0x8a, 0x26, 0xe3, 0xc0, 0x43, 0x6d, 0x58, 0x8b, 0x66, 0x69, 0xca, 0x65, 0x35, 0x04, + 0xd0, 0x38, 0xa3, 0xbf, 0xe3, 0xeb, 0x55, 0xae, 0x67, 0x9e, 0x5d, 0xe0, 0x1f, 0xcc, 0xf5, 0x58, + 0xc7, 0x05, 0x66, 0x42, 0x0e, 0x56, 0x50, 0xe0, 0x7e, 0xbe, 0x06, 0x1e, 0xe7, 0xd8, 0xf3, 0x7a, + 0x50, 0xe3, 0x26, 0x66, 0x5a, 0x95, 0xae, 0xcd, 0xa0, 0x16, 0xf8, 0x3c, 0x05, 0x35, 0x66, 0x05, + 0x75, 0x7e, 0x32, 0xce, 0xe0, 0x14, 0x34, 0x9e, 0x0c, 0xbe, 0xfa, 0xef, 0x9e, 0xf7, 0xe5, 0xbb, + 0x3d, 0xef, 0xab, 0x77, 0x7b, 0xde, 0x7f, 0xde, 0xed, 0xad, 0xfc, 0xf5, 0x7f, 0x7b, 0xde, 0xaf, + 0x0e, 0xad, 0x7f, 0x86, 0x53, 0xcc, 0xf2, 0xf8, 0x9a, 0xe6, 0xf1, 0x24, 0x4e, 0x35, 0x91, 0x92, + 0xa3, 0xec, 0x8b, 0xc9, 0x51, 0x36, 0x3c, 0x12, 0xf5, 0x7b, 0xd8, 0x10, 0xbf, 0x03, 0x1f, 0xfd, + 0x3f, 0x00, 0x00, 0xff, 0xff, 0x00, 0x7f, 0x8a, 0xc1, 0x80, 0x14, 0x00, 0x00, } func (m *ShardsMetadata) Marshal() (dAtA []byte, err error) { @@ -3960,11 +3951,6 @@ func (m *ReaderBuildParam) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.TombstoneApplyPolicy != 0 { i = encodeVarintShard(dAtA, i, uint64(m.TombstoneApplyPolicy)) i-- - dAtA[i] = 0x20 - } - if m.ScanType != 0 { - i = encodeVarintShard(dAtA, i, uint64(m.ScanType)) - i-- dAtA[i] = 0x18 } if m.Expr != nil { @@ -4825,9 +4811,6 @@ func (m *ReaderBuildParam) ProtoSize() (n int) { l = m.Expr.ProtoSize() n += 1 + l + sovShard(uint64(l)) } - if m.ScanType != 0 { - n += 1 + sovShard(uint64(m.ScanType)) - } if m.TombstoneApplyPolicy != 0 { n += 1 + sovShard(uint64(m.TombstoneApplyPolicy)) } @@ -8807,25 +8790,6 @@ func (m *ReaderBuildParam) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ScanType", wireType) - } - m.ScanType = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShard - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ScanType |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TombstoneApplyPolicy", wireType) } diff --git a/pkg/vm/engine/disttae/engine.go b/pkg/vm/engine/disttae/engine.go index af449e1a7cdd..892a60e44c6f 100644 --- a/pkg/vm/engine/disttae/engine.go +++ b/pkg/vm/engine/disttae/engine.go @@ -631,16 +631,6 @@ func (e *Engine) Hints() (h engine.Hints) { return } -func determineScanType(relData engine.RelData, readerNum int) (scanType int) { - scanType = NORMAL - if relData.DataCnt() < readerNum*SMALLSCAN_THRESHOLD || readerNum == 1 { - scanType = SMALL - } else if (readerNum * LARGESCAN_THRESHOLD) <= relData.DataCnt() { - scanType = LARGE - } - return -} - func (e *Engine) BuildBlockReaders( ctx context.Context, p any, @@ -667,7 +657,6 @@ func (e *Engine) BuildBlockReaders( return nil, err } - scanType := determineScanType(relData, newNum) mod := blkCnt % newNum divide := blkCnt / newNum for i := 0; i < newNum; i++ { @@ -694,7 +683,6 @@ func (e *Engine) BuildBlockReaders( if err != nil { return nil, err } - rd.scanType = scanType rds = append(rds, rd) } return rds, nil diff --git a/pkg/vm/engine/disttae/reader.go b/pkg/vm/engine/disttae/reader.go index d85a55093757..a4a21e008750 100644 --- a/pkg/vm/engine/disttae/reader.go +++ b/pkg/vm/engine/disttae/reader.go @@ -357,7 +357,10 @@ func (r *reader) Read( } var policy fileservice.Policy - if r.scanType == LARGE || r.scanType == NORMAL { + if r.readBlockCnt == 0 { + r.smallScanThreshHold = GetSmallScanThreshHold() + } + if r.readBlockCnt > r.smallScanThreshHold { policy = fileservice.SkipMemoryCacheWrites } @@ -381,6 +384,7 @@ func (r *reader) Read( if err != nil { return false, err } + r.readBlockCnt++ if filter.Valid { // we collect mem cache hit related statistics info for blk read here diff --git a/pkg/vm/engine/disttae/txn_table.go b/pkg/vm/engine/disttae/txn_table.go index 47f500bbb758..129b91112cba 100644 --- a/pkg/vm/engine/disttae/txn_table.go +++ b/pkg/vm/engine/disttae/txn_table.go @@ -1747,7 +1747,6 @@ func (tbl *txnTable) BuildReaders( } } - scanType := determineScanType(relData, newNum) def := tbl.GetTableDef(ctx) mod := blkCnt % newNum divide := blkCnt / newNum @@ -1777,8 +1776,6 @@ func (tbl *txnTable) BuildReaders( if err != nil { return nil, err } - - rd.scanType = scanType rds = append(rds, rd) } return rds, nil diff --git a/pkg/vm/engine/disttae/txn_table_sharding.go b/pkg/vm/engine/disttae/txn_table_sharding.go index 2974a5beb826..b62abf22537b 100644 --- a/pkg/vm/engine/disttae/txn_table_sharding.go +++ b/pkg/vm/engine/disttae/txn_table_sharding.go @@ -430,7 +430,6 @@ type shardingLocalReader struct { //relation data to distribute to remote CN which holds shard's partition state. remoteRelData engine.RelData remoteTombApplyPolicy engine.TombstoneApplyPolicy - remoteScanType int } // TODO:: @@ -478,7 +477,6 @@ func (r *shardingLocalReader) Read( func(param *shard.ReadParam) { param.ReaderBuildParam.RelData = relData param.ReaderBuildParam.Expr = expr - param.ReaderBuildParam.ScanType = int32(r.remoteScanType) param.ReaderBuildParam.TombstoneApplyPolicy = int32(r.remoteTombApplyPolicy) }, func(resp []byte) { @@ -626,7 +624,6 @@ func (tbl *txnTableDelegate) BuildShardingReaders( } } - scanType := determineScanType(relData, newNum) mod := blkCnt % newNum divide := blkCnt / newNum current := 0 @@ -647,7 +644,6 @@ func (tbl *txnTableDelegate) BuildShardingReaders( tblDelegate: tbl, //remoteRelData: remoteRelData, remoteTombApplyPolicy: engine.Policy_SkipUncommitedInMemory | engine.Policy_SkipUncommitedS3, - remoteScanType: scanType, } if localRelData.DataCnt() > 0 { @@ -672,7 +668,6 @@ func (tbl *txnTableDelegate) BuildShardingReaders( if err != nil { return nil, err } - lrd.scanType = scanType srd.lrd = lrd } diff --git a/pkg/vm/engine/disttae/types.go b/pkg/vm/engine/disttae/types.go index 56687c6d65ae..370d156da9a8 100644 --- a/pkg/vm/engine/disttae/types.go +++ b/pkg/vm/engine/disttae/types.go @@ -55,12 +55,15 @@ import ( "github.com/matrixorigin/matrixone/pkg/vm/process" ) -const ( - PREFETCH_THRESHOLD = 256 - PREFETCH_ROUNDS = 24 - SMALLSCAN_THRESHOLD = 100 - LARGESCAN_THRESHOLD = 1500 -) +func GetSmallScanThreshHold() uint64 { + if ncpu > 32 { + return 100 + } + if ncpu > 16 { + return 200 + } + return 400 +} const ( INSERT = iota @@ -117,12 +120,6 @@ func noteSplitAlter(note string) (bool, int, uint64, string) { panic("bad format of alter note") } -const ( - SMALL = iota - NORMAL - LARGE -) - const ( MO_DATABASE_ID_NAME_IDX = 1 MO_DATABASE_ID_ACCOUNT_IDX = 2 @@ -921,9 +918,9 @@ type reader struct { isTombstone bool source engine.DataSource - memFilter MemPKFilter - - scanType int + memFilter MemPKFilter + readBlockCnt uint64 + smallScanThreshHold uint64 } type mergeReader struct { diff --git a/pkg/vm/engine/test/reader_test.go b/pkg/vm/engine/test/reader_test.go index cb2779fac9a4..306e6a67fabc 100644 --- a/pkg/vm/engine/test/reader_test.go +++ b/pkg/vm/engine/test/reader_test.go @@ -782,7 +782,6 @@ func Test_ShardingRemoteReader(t *testing.T) { data, err := relData.MarshalBinary() require.NoError(t, err) readerBuildParam.ReaderBuildParam.RelData = data - readerBuildParam.ReaderBuildParam.ScanType = disttae.SMALL readerBuildParam.ReaderBuildParam.TombstoneApplyPolicy = int32(engine.Policy_SkipUncommitedInMemory | engine.Policy_SkipUncommitedS3) res, err := disttae.HandleShardingReadBuildReader( diff --git a/proto/shard.proto b/proto/shard.proto index 3c76182dcb8b..05fa7569516c 100644 --- a/proto/shard.proto +++ b/proto/shard.proto @@ -271,8 +271,7 @@ message KeyParam { message ReaderBuildParam { bytes relData = 1; plan.Expr expr = 2; - int32 scanType = 3; - int32 tombstoneApplyPolicy = 4; + int32 tombstoneApplyPolicy = 3; } message ReaderBuildResult { From d22360d7c8d58c653efa7370e67d50c9d94349fc Mon Sep 17 00:00:00 2001 From: triump2020 <63033222+triump2020@users.noreply.github.com> Date: Fri, 20 Sep 2024 16:49:52 +0800 Subject: [PATCH 2/3] [#15201]Remove BVT tag (#18876) 1. Remove the bvt tag Approved by: @heni02 --- test/distributed/cases/join/leftjoin.result | 23 +++++++++------------ test/distributed/cases/join/leftjoin.test | 3 --- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/test/distributed/cases/join/leftjoin.result b/test/distributed/cases/join/leftjoin.result index 97de3cad164b..cbd11fb45b09 100644 --- a/test/distributed/cases/join/leftjoin.result +++ b/test/distributed/cases/join/leftjoin.result @@ -22,15 +22,12 @@ mo_ctl(dn, flush, d1.t2) select mo_ctl('dn', 'flush', 'd1.t3'); mo_ctl(dn, flush, d1.t3) {\n "method": "Flush",\n "result": [\n {\n "returnStr": "OK"\n }\n ]\n}\n -select Sleep(5); -sleep(5) -0 explain select t2.c1 from t2 left join t1 on t1.c1 =t2.c1; -QUERY PLAN +TP QURERY PLAN Project -> Table Scan on d1.t2 explain select t1.c1,t2.c1 from t2 left join t1 on t1.c1 =t2.c1; -QUERY PLAN +TP QURERY PLAN Project -> Join Join Type: LEFT hashOnPK @@ -38,13 +35,13 @@ Project -> Table Scan on d1.t2 -> Table Scan on d1.t1 explain select t3.c1 from t3 left join t1 on t1.c1 =t3.c1 and t1.c1 > t3.c2 where t3.c1<10; -QUERY PLAN +TP QURERY PLAN Project -> Table Scan on d1.t3 Filter Cond: (t3.c1 < 10) Block Filter Cond: (t3.c1 < 10) explain select t1.c1,t3.c1 from t3 left join t1 on t1.c1 =t3.c1 where t3.c1<10; -QUERY PLAN +TP QURERY PLAN Project -> Join Join Type: LEFT hashOnPK @@ -56,11 +53,11 @@ Project Filter Cond: (t1.c1 < 10) Block Filter Cond: (t1.c1 < 10) explain select t2.c1 from t2 left join t1 on t1.c1 =t2.c2; -QUERY PLAN +TP QURERY PLAN Project -> Table Scan on d1.t2 explain select t3.c1 from t3 left join t2 on t3.c2=t2.c2; -QUERY PLAN +TP QURERY PLAN Project -> Join Join Type: LEFT @@ -68,11 +65,11 @@ Project -> Table Scan on d1.t3 -> Table Scan on d1.t2 explain select t2.c1 from t2 left join (select t3.c1,t3.c2 from t1 join t3 on t1.c1=t3.c1) v1 on t2.c1 =v1.c1 and t2.c2=v1.c2; -QUERY PLAN +TP QURERY PLAN Project -> Table Scan on d1.t2 explain select t2.c1 from t2 left join (select t3.c1,t3.c2 from t1 join t3 on t1.c1=t3.c1 limit 5000) v1 on t2.c1 =v1.c1; -QUERY PLAN +TP QURERY PLAN Project -> Join Join Type: LEFT @@ -150,9 +147,9 @@ insert into routines values('specific_name01','routine_catalog01','specific_name insert into routines values('specific_name02','routine_catalog02','specific_name01','routine_name01','rrr01','data',10,10,10,20,20,'character_set_name','collation','dtd_identifier01','111','routine_definition01','1','external_language01','aaa','bbb','sql_data','sql_path01','111','2020-10-10 12:12:12.000','2021-10-10 12:12:12.000','sql_mode01','routine_comment01','definer01','character_set_client','collation_connection','database_collation'); insert into routines values('specific_name01','routine_catalog01','specific_name02','routine_name02','rrr02','data',10,10,10,20,20,'character_set_name','collation','dtd_identifier01','111','routine_definition01','1','external_language01','aaa','bbb','sql_data','sql_path01','111','2020-10-10 12:12:12.000','2021-10-10 12:12:12.000','sql_mode01','routine_comment01','definer01','character_set_client','collation_connection','database_collation'); SELECT DISTINCT ROUTINE_SCHEMA, ROUTINE_NAME, PARAMS.PARAMETER FROM ROUTINES LEFT JOIN ( SELECT SPECIFIC_SCHEMA, SPECIFIC_NAME, GROUP_CONCAT(CONCAT(DATA_TYPE, ' ', PARAMETER_NAME) ORDER BY ORDINAL_POSITION SEPARATOR ', ') PARAMETER, ROUTINE_TYPE FROM PARAMETERS GROUP BY SPECIFIC_SCHEMA, SPECIFIC_NAME, ROUTINE_TYPE ) PARAMS ON ROUTINES.ROUTINE_SCHEMA = PARAMS.SPECIFIC_SCHEMA AND ROUTINES.ROUTINE_NAME = PARAMS.SPECIFIC_NAME AND ROUTINES.ROUTINE_TYPE = PARAMS.ROUTINE_TYPE WHERE ROUTINE_SCHEMA = 'specific_name01' ORDER BY ROUTINE_SCHEMA; -routine_schema routine_name parameter +ROUTINE_SCHEMA ROUTINE_NAME PARAMETER specific_name01 routine_name02 null specific_name01 routine_name01 data_type01 parametername01 drop table routines; drop table parameters; -drop database if exists d1; \ No newline at end of file +drop database if exists d1; diff --git a/test/distributed/cases/join/leftjoin.test b/test/distributed/cases/join/leftjoin.test index 2482d6bb2ee1..3edebcf400b6 100644 --- a/test/distributed/cases/join/leftjoin.test +++ b/test/distributed/cases/join/leftjoin.test @@ -19,7 +19,6 @@ select mo_ctl('dn', 'flush', 'd1.t1'); select mo_ctl('dn', 'flush', 'd1.t2'); -- @separator:table select mo_ctl('dn', 'flush', 'd1.t3'); -select Sleep(5); -- @separator:table explain select t2.c1 from t2 left join t1 on t1.c1 =t2.c1; -- @separator:table @@ -33,9 +32,7 @@ explain select t2.c1 from t2 left join t1 on t1.c1 =t2.c2; -- @separator:table explain select t3.c1 from t3 left join t2 on t3.c2=t2.c2; -- @separator:table --- @bvt:issue#15201 explain select t2.c1 from t2 left join (select t3.c1,t3.c2 from t1 join t3 on t1.c1=t3.c1) v1 on t2.c1 =v1.c1 and t2.c2=v1.c2; --- @bvt:issue -- @separator:table explain select t2.c1 from t2 left join (select t3.c1,t3.c2 from t1 join t3 on t1.c1=t3.c1 limit 5000) v1 on t2.c1 =v1.c1; drop table if exists parameters; From 949c826ba247bb978878f620a6dca5986d1d9536 Mon Sep 17 00:00:00 2001 From: LiuBo Date: Fri, 20 Sep 2024 17:44:45 +0800 Subject: [PATCH 3/3] [bug] fix ut TestNodeGossip (#18882) fix ut TestNodeGossip Approved by: @sukki37 --- pkg/gossip/node_test.go | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/gossip/node_test.go b/pkg/gossip/node_test.go index 85860e4b2499..af4087e61199 100644 --- a/pkg/gossip/node_test.go +++ b/pkg/gossip/node_test.go @@ -16,6 +16,8 @@ package gossip import ( "context" + "fmt" + "math/rand" "testing" "time" @@ -23,18 +25,25 @@ import ( "github.com/stretchr/testify/assert" ) +func getRandomPort() int { + rand.New(rand.NewSource(time.Now().UnixNano())) + return rand.Intn(65535-1024) + 1024 +} + func TestNodeGossip(t *testing.T) { defer leaktest.AfterTest(t)() ctx := context.Background() + svcPort1 := getRandomPort() + cachePort1 := getRandomPort() n1, err := NewNode(ctx, "n1", WithListenAddrFn(func() string { - return "127.0.0.1:38001" + return fmt.Sprintf("127.0.0.1:%d", svcPort1) }), WithServiceAddrFn(func() string { - return "127.0.0.1:38001" + return fmt.Sprintf("127.0.0.1:%d", svcPort1) }), WithCacheServerAddrFn(func() string { - return "127.0.0.1:38101" + return fmt.Sprintf("127.0.0.1:%d", cachePort1) })) assert.NoError(t, err) assert.NotNil(t, n1) @@ -46,17 +55,19 @@ func TestNodeGossip(t *testing.T) { err = n1.Join(nil) assert.Equal(t, 1, n1.NumMembers()) assert.NotNil(t, n1.DistKeyCacheGetter()()) - assert.NoError(t, err) + + svcPort2 := getRandomPort() + cachePort2 := getRandomPort() n2, err := NewNode(ctx, "n2", WithListenAddrFn(func() string { - return "127.0.0.1:38002" + return fmt.Sprintf("127.0.0.1:%d", svcPort2) }), WithServiceAddrFn(func() string { - return "127.0.0.1:38002" + return fmt.Sprintf("127.0.0.1:%d", svcPort2) }), WithCacheServerAddrFn(func() string { - return "127.0.0.1:38201" + return fmt.Sprintf("127.0.0.1:%d", cachePort2) })) assert.NoError(t, err) assert.NotNil(t, n2) @@ -65,7 +76,7 @@ func TestNodeGossip(t *testing.T) { defer func() { assert.NoError(t, n2.Leave(time.Millisecond*300)) }() - err = n2.Join([]string{"127.0.0.1:38001"}) + err = n2.Join([]string{fmt.Sprintf("127.0.0.1:%d", svcPort1)}) assert.NoError(t, err) assert.Equal(t, 2, n2.NumMembers()) assert.NotNil(t, n2.DistKeyCacheGetter()())