feat(eventstore): increase parallel write capabilities (#5940)

This implementation increases parallel write capabilities of the eventstore.
Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and  [06](https://zitadel.com/docs/support/advisory/a10006).
The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`.
If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
Silvan
2023-10-19 12:19:10 +02:00
committed by GitHub
parent 259faba3f0
commit b5564572bc
791 changed files with 30326 additions and 43202 deletions
+2 -14
View File
@@ -8,27 +8,15 @@ import (
)
func (s *Server) ListFailedEvents(ctx context.Context, _ *system_pb.ListFailedEventsRequest) (*system_pb.ListFailedEventsResponse, error) {
failedEventsOld, err := s.administrator.GetFailedEvents(ctx, "")
if err != nil {
return nil, err
}
convertedOld := FailedEventsViewToPb(failedEventsOld)
failedEvents, err := s.query.SearchFailedEvents(ctx, new(query.FailedEventSearchQueries))
if err != nil {
return nil, err
}
convertedNew := FailedEventsToPb(s.database, failedEvents)
return &system_pb.ListFailedEventsResponse{Result: append(convertedOld, convertedNew...)}, nil
return &system_pb.ListFailedEventsResponse{Result: FailedEventsToPb(s.database, failedEvents)}, nil
}
func (s *Server) RemoveFailedEvent(ctx context.Context, req *system_pb.RemoveFailedEventRequest) (*system_pb.RemoveFailedEventResponse, error) {
var err error
if req.Database != s.database {
err = s.administrator.RemoveFailedEvent(ctx, RemoveFailedEventRequestToModel(req))
} else {
err = s.query.RemoveFailedEvent(ctx, req.ViewName, req.InstanceId, req.FailedSequence)
}
err := s.query.RemoveFailedEvent(ctx, req.ViewName, req.InstanceId, req.FailedSequence)
if err != nil {
return nil, err
}
@@ -4,33 +4,9 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/view/model"
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
)
func FailedEventsViewToPb(failedEvents []*model.FailedEvent) []*system_pb.FailedEvent {
events := make([]*system_pb.FailedEvent, len(failedEvents))
for i, failedEvent := range failedEvents {
events[i] = FailedEventViewToPb(failedEvent)
}
return events
}
func FailedEventViewToPb(failedEvent *model.FailedEvent) *system_pb.FailedEvent {
var lastFailed *timestamppb.Timestamp
if !failedEvent.LastFailed.IsZero() {
lastFailed = timestamppb.New(failedEvent.LastFailed)
}
return &system_pb.FailedEvent{
Database: failedEvent.Database,
ViewName: failedEvent.ViewName,
FailedSequence: failedEvent.FailedSequence,
FailureCount: failedEvent.FailureCount,
ErrorMessage: failedEvent.ErrMsg,
LastFailed: lastFailed,
}
}
func FailedEventsToPb(database string, failedEvents *query.FailedEvents) []*system_pb.FailedEvent {
events := make([]*system_pb.FailedEvent, len(failedEvents.FailedEvents))
for i, failedEvent := range failedEvents.FailedEvents {
@@ -53,12 +29,3 @@ func FailedEventToPb(database string, failedEvent *query.FailedEvent) *system_pb
LastFailed: lastFailed,
}
}
func RemoveFailedEventRequestToModel(req *system_pb.RemoveFailedEventRequest) *model.FailedEvent {
return &model.FailedEvent{
Database: req.Database,
ViewName: req.ViewName,
FailedSequence: req.FailedSequence,
InstanceID: req.InstanceId,
}
}
@@ -1,99 +0,0 @@
package system_test
import (
"testing"
"time"
system_grpc "github.com/zitadel/zitadel/internal/api/grpc/system"
"github.com/zitadel/zitadel/internal/test"
"github.com/zitadel/zitadel/internal/view/model"
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
)
func TestFailedEventsToPbFields(t *testing.T) {
type args struct {
failedEvents []*model.FailedEvent
}
tests := []struct {
name string
args args
}{
{
name: "all fields",
args: args{
failedEvents: []*model.FailedEvent{
{
Database: "admin",
ViewName: "users",
FailedSequence: 456,
FailureCount: 5,
LastFailed: time.Now(),
ErrMsg: "some error",
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := system_grpc.FailedEventsViewToPb(tt.args.failedEvents)
for _, g := range got {
test.AssertFieldsMapped(t, g)
}
})
}
}
func TestFailedEventToPbFields(t *testing.T) {
type args struct {
failedEvent *model.FailedEvent
}
tests := []struct {
name string
args args
}{
{
"all fields",
args{
failedEvent: &model.FailedEvent{
Database: "admin",
ViewName: "users",
FailedSequence: 456,
FailureCount: 5,
LastFailed: time.Now(),
ErrMsg: "some error",
},
},
},
}
for _, tt := range tests {
converted := system_grpc.FailedEventViewToPb(tt.args.failedEvent)
test.AssertFieldsMapped(t, converted)
}
}
func TestRemoveFailedEventRequestToModelFields(t *testing.T) {
type args struct {
req *system_pb.RemoveFailedEventRequest
}
tests := []struct {
name string
args args
}{
{
"all fields",
args{
req: &system_pb.RemoveFailedEventRequest{
Database: "admin",
ViewName: "users",
FailedSequence: 456,
InstanceId: "instanceID",
},
},
},
}
for _, tt := range tests {
converted := system_grpc.RemoveFailedEventRequestToModel(tt.args.req)
test.AssertFieldsMapped(t, converted, "FailureCount", "LastFailed", "ErrMsg")
}
}
+2 -2
View File
@@ -103,7 +103,7 @@ func (s *Server) ListIAMMembers(ctx context.Context, req *system_pb.ListIAMMembe
return nil, err
}
return &system_pb.ListIAMMembersResponse{
Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
//TODO: resource owner of user of the member instead of the membership resource owner
Result: member.MembersToPb("", res.Members),
}, nil
@@ -146,7 +146,7 @@ func (s *Server) ListDomains(ctx context.Context, req *system_pb.ListDomainsRequ
}
return &system_pb.ListDomainsResponse{
Result: instance_grpc.DomainsToPb(domains.Domains),
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.Timestamp),
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.LastRun),
}, nil
}
-4
View File
@@ -3,7 +3,6 @@ package system
import (
"google.golang.org/grpc"
"github.com/zitadel/zitadel/internal/admin/repository"
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/grpc/server"
@@ -23,7 +22,6 @@ type Server struct {
database string
command *command.Commands
query *query.Queries
administrator repository.AdministratorRepository
defaultInstance command.InstanceSetup
externalDomain string
}
@@ -35,7 +33,6 @@ type Config struct {
func CreateServer(
command *command.Commands,
query *query.Queries,
repo repository.Repository,
database string,
defaultInstance command.InstanceSetup,
externalDomain string,
@@ -43,7 +40,6 @@ func CreateServer(
return &Server{
command: command,
query: query,
administrator: repo,
database: database,
defaultInstance: defaultInstance,
externalDomain: externalDomain,
+3 -16
View File
@@ -8,28 +8,15 @@ import (
)
func (s *Server) ListViews(ctx context.Context, _ *system_pb.ListViewsRequest) (*system_pb.ListViewsResponse, error) {
currentSequences, err := s.query.SearchCurrentSequences(ctx, new(query.CurrentSequencesSearchQueries))
currentSequences, err := s.query.SearchCurrentStates(ctx, new(query.CurrentStateSearchQueries))
if err != nil {
return nil, err
}
convertedCurrentSequences := CurrentSequencesToPb(s.database, currentSequences)
views, err := s.administrator.GetViews("")
if err != nil {
return nil, err
}
convertedViews := ViewsToPb(views)
convertedCurrentSequences = append(convertedCurrentSequences, convertedViews...)
return &system_pb.ListViewsResponse{Result: convertedCurrentSequences}, nil
return &system_pb.ListViewsResponse{Result: CurrentSequencesToPb(s.database, currentSequences)}, nil
}
func (s *Server) ClearView(ctx context.Context, req *system_pb.ClearViewRequest) (*system_pb.ClearViewResponse, error) {
var err error
if req.Database != s.database {
err = s.administrator.ClearView(ctx, req.Database, req.ViewName)
} else {
err = s.query.ClearCurrentSequence(ctx, req.ViewName)
}
err := s.query.ClearCurrentSequence(ctx, req.ViewName)
if err != nil {
return nil, err
}
+6 -25
View File
@@ -4,41 +4,22 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/zitadel/zitadel/internal/query"
"github.com/zitadel/zitadel/internal/view/model"
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
)
func ViewsToPb(views []*model.View) []*system_pb.View {
v := make([]*system_pb.View, len(views))
for i, view := range views {
v[i] = ViewToPb(view)
}
return v
}
func ViewToPb(view *model.View) *system_pb.View {
return &system_pb.View{
Database: view.Database,
ViewName: view.ViewName,
LastSuccessfulSpoolerRun: timestamppb.New(view.LastSuccessfulSpoolerRun),
ProcessedSequence: view.CurrentSequence,
EventTimestamp: timestamppb.New(view.EventTimestamp),
}
}
func CurrentSequencesToPb(database string, currentSequences *query.CurrentSequences) []*system_pb.View {
v := make([]*system_pb.View, len(currentSequences.CurrentSequences))
for i, currentSequence := range currentSequences.CurrentSequences {
func CurrentSequencesToPb(database string, currentSequences *query.CurrentStates) []*system_pb.View {
v := make([]*system_pb.View, len(currentSequences.CurrentStates))
for i, currentSequence := range currentSequences.CurrentStates {
v[i] = CurrentSequenceToPb(database, currentSequence)
}
return v
}
func CurrentSequenceToPb(database string, currentSequence *query.CurrentSequence) *system_pb.View {
func CurrentSequenceToPb(database string, currentSequence *query.CurrentState) *system_pb.View {
return &system_pb.View{
Database: database,
ViewName: currentSequence.ProjectionName,
ProcessedSequence: currentSequence.CurrentSequence,
LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.Timestamp),
ProcessedSequence: currentSequence.Sequence,
LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.LastRun),
}
}