mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: add Prometheus metrics for boundary log drop reporting (#22521)
Add Prometheus metrics to the boundary log proxy for observability: - batches_dropped_total (reason: buffer_full, forward_failed) - logs_dropped_total (reason: buffer_full, forward_failed, boundary_channel_full, boundary_batch_full) - batches_forwarded_total Also add BoundaryStatus to the BoundaryMessage envelope so boundary can report dropped log counts as a separate wire message. The agent records these as Prometheus metrics, making boundary-side data loss visible. Backwards compatibility for older versions of boundary is maintained. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Zach
@@ -34,6 +34,7 @@ type BoundaryMessage struct {
|
||||
// Types that are assignable to Msg:
|
||||
//
|
||||
// *BoundaryMessage_Logs
|
||||
// *BoundaryMessage_Status
|
||||
Msg isBoundaryMessage_Msg `protobuf_oneof:"msg"`
|
||||
}
|
||||
|
||||
@@ -83,6 +84,13 @@ func (x *BoundaryMessage) GetLogs() *proto.ReportBoundaryLogsRequest {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *BoundaryMessage) GetStatus() *BoundaryStatus {
|
||||
if x, ok := x.GetMsg().(*BoundaryMessage_Status); ok {
|
||||
return x.Status
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isBoundaryMessage_Msg interface {
|
||||
isBoundaryMessage_Msg()
|
||||
}
|
||||
@@ -91,8 +99,75 @@ type BoundaryMessage_Logs struct {
|
||||
Logs *proto.ReportBoundaryLogsRequest `protobuf:"bytes,1,opt,name=logs,proto3,oneof"`
|
||||
}
|
||||
|
||||
type BoundaryMessage_Status struct {
|
||||
Status *BoundaryStatus `protobuf:"bytes,2,opt,name=status,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*BoundaryMessage_Logs) isBoundaryMessage_Msg() {}
|
||||
|
||||
func (*BoundaryMessage_Status) isBoundaryMessage_Msg() {}
|
||||
|
||||
// BoundaryStatus carries operational metadata from boundary to the agent.
|
||||
// The agent records these values as Prometheus metrics. This message is
|
||||
// never forwarded to coderd.
|
||||
type BoundaryStatus struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
// Logs dropped because boundary's internal channel buffer was full.
|
||||
DroppedChannelFull int64 `protobuf:"varint,1,opt,name=dropped_channel_full,json=droppedChannelFull,proto3" json:"dropped_channel_full,omitempty"`
|
||||
// Logs dropped because boundary's batch buffer was full after a
|
||||
// failed flush attempt.
|
||||
DroppedBatchFull int64 `protobuf:"varint,2,opt,name=dropped_batch_full,json=droppedBatchFull,proto3" json:"dropped_batch_full,omitempty"`
|
||||
}
|
||||
|
||||
func (x *BoundaryStatus) Reset() {
|
||||
*x = BoundaryStatus{}
|
||||
if protoimpl.UnsafeEnabled {
|
||||
mi := &file_agent_boundarylogproxy_codec_boundary_proto_msgTypes[1]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
}
|
||||
|
||||
func (x *BoundaryStatus) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*BoundaryStatus) ProtoMessage() {}
|
||||
|
||||
func (x *BoundaryStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_agent_boundarylogproxy_codec_boundary_proto_msgTypes[1]
|
||||
if protoimpl.UnsafeEnabled && x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use BoundaryStatus.ProtoReflect.Descriptor instead.
|
||||
func (*BoundaryStatus) Descriptor() ([]byte, []int) {
|
||||
return file_agent_boundarylogproxy_codec_boundary_proto_rawDescGZIP(), []int{1}
|
||||
}
|
||||
|
||||
func (x *BoundaryStatus) GetDroppedChannelFull() int64 {
|
||||
if x != nil {
|
||||
return x.DroppedChannelFull
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *BoundaryStatus) GetDroppedBatchFull() int64 {
|
||||
if x != nil {
|
||||
return x.DroppedBatchFull
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
var File_agent_boundarylogproxy_codec_boundary_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_agent_boundarylogproxy_codec_boundary_proto_rawDesc = []byte{
|
||||
@@ -102,17 +177,29 @@ var file_agent_boundarylogproxy_codec_boundary_proto_rawDesc = []byte{
|
||||
0x6f, 0x64, 0x65, 0x72, 0x2e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67,
|
||||
0x70, 0x72, 0x6f, 0x78, 0x79, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x2e, 0x76, 0x31, 0x1a, 0x17,
|
||||
0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x61, 0x67, 0x65, 0x6e,
|
||||
0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x59, 0x0a, 0x0f, 0x42, 0x6f, 0x75, 0x6e, 0x64,
|
||||
0x61, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x6c, 0x6f,
|
||||
0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x72,
|
||||
0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74,
|
||||
0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x42, 0x05, 0x0a, 0x03, 0x6d,
|
||||
0x73, 0x67, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
|
||||
0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f,
|
||||
0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x6c, 0x6f,
|
||||
0x67, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x62, 0x06, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x33,
|
||||
0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa4, 0x01, 0x0a, 0x0f, 0x42, 0x6f, 0x75, 0x6e,
|
||||
0x64, 0x61, 0x72, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x04, 0x6c,
|
||||
0x6f, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x64, 0x65,
|
||||
0x72, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x76, 0x32, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x72,
|
||||
0x74, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x4c, 0x6f, 0x67, 0x73, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x04, 0x6c, 0x6f, 0x67, 0x73, 0x12, 0x49, 0x0a, 0x06,
|
||||
0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x63,
|
||||
0x6f, 0x64, 0x65, 0x72, 0x2e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67,
|
||||
0x70, 0x72, 0x6f, 0x78, 0x79, 0x2e, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x2e, 0x76, 0x31, 0x2e, 0x42,
|
||||
0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52,
|
||||
0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x05, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x22, 0x70,
|
||||
0x0a, 0x0e, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73,
|
||||
0x12, 0x30, 0x0a, 0x14, 0x64, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x63, 0x68, 0x61, 0x6e,
|
||||
0x6e, 0x65, 0x6c, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x12,
|
||||
0x64, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x46, 0x75,
|
||||
0x6c, 0x6c, 0x12, 0x2c, 0x0a, 0x12, 0x64, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x5f, 0x62, 0x61,
|
||||
0x74, 0x63, 0x68, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10,
|
||||
0x64, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x75, 0x6c, 0x6c,
|
||||
0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63,
|
||||
0x6f, 0x64, 0x65, 0x72, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x2f, 0x76, 0x32, 0x2f, 0x61, 0x67,
|
||||
0x65, 0x6e, 0x74, 0x2f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x72, 0x79, 0x6c, 0x6f, 0x67, 0x70,
|
||||
0x72, 0x6f, 0x78, 0x79, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,
|
||||
0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -127,18 +214,20 @@ func file_agent_boundarylogproxy_codec_boundary_proto_rawDescGZIP() []byte {
|
||||
return file_agent_boundarylogproxy_codec_boundary_proto_rawDescData
|
||||
}
|
||||
|
||||
var file_agent_boundarylogproxy_codec_boundary_proto_msgTypes = make([]protoimpl.MessageInfo, 1)
|
||||
var file_agent_boundarylogproxy_codec_boundary_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
|
||||
var file_agent_boundarylogproxy_codec_boundary_proto_goTypes = []interface{}{
|
||||
(*BoundaryMessage)(nil), // 0: coder.boundarylogproxy.codec.v1.BoundaryMessage
|
||||
(*proto.ReportBoundaryLogsRequest)(nil), // 1: coder.agent.v2.ReportBoundaryLogsRequest
|
||||
(*BoundaryStatus)(nil), // 1: coder.boundarylogproxy.codec.v1.BoundaryStatus
|
||||
(*proto.ReportBoundaryLogsRequest)(nil), // 2: coder.agent.v2.ReportBoundaryLogsRequest
|
||||
}
|
||||
var file_agent_boundarylogproxy_codec_boundary_proto_depIdxs = []int32{
|
||||
1, // 0: coder.boundarylogproxy.codec.v1.BoundaryMessage.logs:type_name -> coder.agent.v2.ReportBoundaryLogsRequest
|
||||
1, // [1:1] is the sub-list for method output_type
|
||||
1, // [1:1] is the sub-list for method input_type
|
||||
1, // [1:1] is the sub-list for extension type_name
|
||||
1, // [1:1] is the sub-list for extension extendee
|
||||
0, // [0:1] is the sub-list for field type_name
|
||||
2, // 0: coder.boundarylogproxy.codec.v1.BoundaryMessage.logs:type_name -> coder.agent.v2.ReportBoundaryLogsRequest
|
||||
1, // 1: coder.boundarylogproxy.codec.v1.BoundaryMessage.status:type_name -> coder.boundarylogproxy.codec.v1.BoundaryStatus
|
||||
2, // [2:2] is the sub-list for method output_type
|
||||
2, // [2:2] is the sub-list for method input_type
|
||||
2, // [2:2] is the sub-list for extension type_name
|
||||
2, // [2:2] is the sub-list for extension extendee
|
||||
0, // [0:2] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_agent_boundarylogproxy_codec_boundary_proto_init() }
|
||||
@@ -159,9 +248,22 @@ func file_agent_boundarylogproxy_codec_boundary_proto_init() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
file_agent_boundarylogproxy_codec_boundary_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
|
||||
switch v := v.(*BoundaryStatus); i {
|
||||
case 0:
|
||||
return &v.state
|
||||
case 1:
|
||||
return &v.sizeCache
|
||||
case 2:
|
||||
return &v.unknownFields
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
file_agent_boundarylogproxy_codec_boundary_proto_msgTypes[0].OneofWrappers = []interface{}{
|
||||
(*BoundaryMessage_Logs)(nil),
|
||||
(*BoundaryMessage_Status)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -169,7 +271,7 @@ func file_agent_boundarylogproxy_codec_boundary_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: file_agent_boundarylogproxy_codec_boundary_proto_rawDesc,
|
||||
NumEnums: 0,
|
||||
NumMessages: 1,
|
||||
NumMessages: 2,
|
||||
NumExtensions: 0,
|
||||
NumServices: 0,
|
||||
},
|
||||
|
||||
@@ -13,5 +13,17 @@ import "agent/proto/agent.proto";
|
||||
message BoundaryMessage {
|
||||
oneof msg {
|
||||
coder.agent.v2.ReportBoundaryLogsRequest logs = 1;
|
||||
BoundaryStatus status = 2;
|
||||
}
|
||||
}
|
||||
|
||||
// BoundaryStatus carries operational metadata from boundary to the agent.
|
||||
// The agent records these values as Prometheus metrics. This message is
|
||||
// never forwarded to coderd.
|
||||
message BoundaryStatus {
|
||||
// Logs dropped because boundary's internal channel buffer was full.
|
||||
int64 dropped_channel_full = 1;
|
||||
// Logs dropped because boundary's batch buffer was full after a
|
||||
// failed flush attempt.
|
||||
int64 dropped_batch_full = 2;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user