Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Modify S3IOBytes statistic property #18795

Merged
merged 11 commits into from
Sep 18, 2024
3 changes: 2 additions & 1 deletion CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@
# pkg/sql
/pkg/sql @aunjgr
/pkg/sql/colexec @m-schen
/pkg/sql/compile @m-schen @ouyuanning @aunjgr @badboynt1
/pkg/sql/compile @m-schen @ouyuanning @aunjgr @badboynt1 @qingxinhome
/pkg/sql/models @qingxinhome
/pkg/sql/parsers @iamlinjunhong
/pkg/sql/plan @ouyuanning @aunjgr @badboynt1
/pkg/sql/plan/function @m-schen
Expand Down
986 changes: 493 additions & 493 deletions pkg/pb/plan/plan.pb.go

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pkg/sql/colexec/table_scan/table_scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ func (tableScan *TableScan) Call(proc *process.Process) (vm.CallResult, error) {
tableScan.ctr.buf)

analyzer.InputBlock()
analyzer.S3IOByte(tableScan.ctr.buf)
analyzer.ScanBytes(tableScan.ctr.buf)
batSize := tableScan.ctr.buf.Size()
tableScan.ctr.maxAllocSize = max(tableScan.ctr.maxAllocSize, batSize)
break
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/compile/analyze_module.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ func applyOpStatsToNode(op *models.PhyOperator, nodes []*plan.Node) {
node.AnalyzeInfo.TimeConsumed += op.OpStats.TotalTimeConsumed
node.AnalyzeInfo.MemorySize += op.OpStats.TotalMemorySize
node.AnalyzeInfo.WaitTimeConsumed += op.OpStats.TotalWaitTimeConsumed
node.AnalyzeInfo.S3IOByte += op.OpStats.TotalS3IOByte
node.AnalyzeInfo.ScanBytes += op.OpStats.TotalScanBytes
node.AnalyzeInfo.NetworkIO += op.OpStats.TotalNetworkIO
node.AnalyzeInfo.InputBlocks += op.OpStats.TotalInputBlocks

Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/compile/analyze_module_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func Test_processPhyScope(t *testing.T) {
TotalInputSize: 2048,
TotalInputBlocks: 0,
TotalOutputSize: 1900,
TotalS3IOByte: 0,
TotalScanBytes: 0,
TotalNetworkIO: 600,
//TotalScanTime: 1500,
//TotalInsertTime: 0,
Expand Down
6 changes: 3 additions & 3 deletions pkg/sql/models/logic_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ const InputSize = "Input Size"
const OutputSize = "Output Size"
const MemorySize = "Memory Size"
const DiskIO = "Disk IO"
const S3IOByte = "S3 IO Byte"
const ScanBytes = "Scan Bytes"
const S3IOInputCount = "S3 IO Input Count"
const S3IOOutputCount = "S3 IO Output Count"
const Network = "Network"
Expand Down Expand Up @@ -238,7 +238,7 @@ func (graphData *GraphData) StatisticsGlobalResource(ctx context.Context) error

//io
gDiskIO := NewStatisticValue(DiskIO, "byte")
gS3IOByte := NewStatisticValue(S3IOByte, "byte")
gS3IOByte := NewStatisticValue(ScanBytes, "byte")
gS3IOInputCount := NewStatisticValue(S3IOInputCount, "count")
gS3IOOutputCount := NewStatisticValue(S3IOOutputCount, "count")

Expand Down Expand Up @@ -286,7 +286,7 @@ func (graphData *GraphData) StatisticsGlobalResource(ctx context.Context) error
if ioValue.Name == DiskIO {
gDiskIO.Value += ioValue.Value
}
if ioValue.Name == S3IOByte {
if ioValue.Name == ScanBytes {
gS3IOByte.Value += ioValue.Value
}
if ioValue.Name == S3IOInputCount {
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/models/phy_plan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func TestPhyPlanJSON(t *testing.T) {
TotalInputSize: 2048,
TotalInputBlocks: 0,
TotalOutputSize: 1900,
TotalS3IOByte: 0,
TotalScanBytes: 0,
TotalNetworkIO: 600,
//TotalScanTime: 1500,
//TotalInsertTime: 2500,
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/models/show_phyplan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestExplainPhyPlan(t *testing.T) {
TotalInputSize: 2048,
TotalInputBlocks: 0,
TotalOutputSize: 1900,
TotalS3IOByte: 0,
TotalScanBytes: 0,
TotalNetworkIO: 600,
//TotalScanTime: 1500,
//TotalInsertTime: 0,
Expand Down
5 changes: 3 additions & 2 deletions pkg/sql/plan/deepcopy.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ package plan
import (
"bytes"

"github.com/matrixorigin/matrixone/pkg/pb/plan"
"golang.org/x/exp/constraints"

"github.com/matrixorigin/matrixone/pkg/pb/plan"
)

func DeepCopyExprList(list []*Expr) []*Expr {
Expand Down Expand Up @@ -1090,7 +1091,7 @@ func DeepCopyAnalyzeInfo(analyzeinfo *plan.AnalyzeInfo) *plan.AnalyzeInfo {
MemorySize: analyzeinfo.GetMemorySize(),
WaitTimeConsumed: analyzeinfo.GetWaitTimeConsumed(),
DiskIO: analyzeinfo.GetDiskIO(),
S3IOByte: analyzeinfo.GetS3IOByte(),
ScanBytes: analyzeinfo.GetScanBytes(),
S3IOInputCount: analyzeinfo.GetS3IOInputCount(),
S3IOOutputCount: analyzeinfo.GetS3IOOutputCount(),
NetworkIO: analyzeinfo.GetNetworkIO(),
Expand Down
6 changes: 3 additions & 3 deletions pkg/sql/plan/explain/marshal_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ const InputSize = "Input Size"
const OutputSize = "Output Size"
const MemorySize = "Memory Size"
const DiskIO = "Disk IO"
const S3IOByte = "S3 IO Byte"
const ScanBytes = "Scan Bytes"
const S3IOInputCount = "S3 IO Input Count"
const S3IOOutputCount = "S3 IO Output Count"
const Network = "Network"
Expand Down Expand Up @@ -791,8 +791,8 @@ func (m MarshalNodeImpl) GetStatistics(ctx context.Context, options *ExplainOpti
Unit: Statistic_Unit_byte, //"byte",
},
{
Name: S3IOByte,
Value: analyzeInfo.S3IOByte,
Name: ScanBytes,
Value: analyzeInfo.ScanBytes,
Unit: Statistic_Unit_byte, //"byte",
},
{
Expand Down
12 changes: 6 additions & 6 deletions pkg/vm/process/operator_analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ type Analyzer interface {
Reset()

InputBlock()
S3IOByte(*batch.Batch) // delete it, unused
ScanBytes(*batch.Batch)
}

// Operator Resource operatorAnalyzer
Expand Down Expand Up @@ -158,13 +158,13 @@ func (opAlyzr *operatorAnalyzer) ChildrenCallStop(start time.Time) {
opAlyzr.childrenCallDuration += time.Since(start)
}

func (opAlyzr *operatorAnalyzer) S3IOByte(bat *batch.Batch) {
func (opAlyzr *operatorAnalyzer) ScanBytes(bat *batch.Batch) {
if opAlyzr.opStats == nil {
panic("operatorAnalyzer.S3IOByte: operatorAnalyzer.opStats is nil")
}

if bat != nil {
opAlyzr.opStats.TotalS3IOByte += int64(bat.Size())
opAlyzr.opStats.TotalScanBytes += int64(bat.Size())
}
}

Expand Down Expand Up @@ -221,7 +221,7 @@ type OperatorStats struct {
TotalOutputSize int64 `json:"TotalOutputSize,omitempty"`
TotalNetworkIO int64 `json:"TotalNetworkIO,omitempty"`
TotalInputBlocks int64 `json:"-"`
TotalS3IOByte int64 `json:"-"`
TotalScanBytes int64 `json:"-"`
OperatorMetrics map[MetricType]int64 `json:"OperatorMetrics,omitempty"`
}

Expand Down Expand Up @@ -277,7 +277,7 @@ func (ps *OperatorStats) String() string {
"InBlock:%d "+
"OutSize:%dbytes "+
"MemSize:%dbytes "+
"S3IOByte:%dbytes "+
"ScanBytes:%dbytes "+
"NetworkIO:%dbytes"+
"%s",
ps.CallNum,
Expand All @@ -289,7 +289,7 @@ func (ps *OperatorStats) String() string {
ps.TotalInputBlocks,
ps.TotalOutputSize,
ps.TotalMemorySize,
ps.TotalS3IOByte,
ps.TotalScanBytes,
ps.TotalNetworkIO,
metricsStr)
}
Expand Down
2 changes: 1 addition & 1 deletion proto/plan.proto
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,7 @@ message AnalyzeInfo {
int64 memory_size = 6;
int64 wait_time_consumed = 7;
int64 diskIO = 8;
int64 s3IO_byte = 9;
int64 scan_bytes = 9;
int64 s3IO_input_count = 10;
int64 s3IO_output_count = 11;
int64 networkIO = 12;
Expand Down
Loading