mirror of
https://github.com/andwati/zitadel.git
synced 2026-11-19 03:35:12 +00:00
feat(eventstore): increase parallel write capabilities (#5940)
This implementation increases parallel write capabilities of the eventstore. Please have a look at the technical advisories: [05](https://zitadel.com/docs/support/advisory/a10005) and [06](https://zitadel.com/docs/support/advisory/a10006). The implementation of eventstore.push is rewritten and stored events are migrated to a new table `eventstore.events2`. If you are using cockroach: make sure that the database user of ZITADEL has `VIEWACTIVITY` grant. This is used to query events.
This commit is contained in:
@@ -8,27 +8,15 @@ import (
|
||||
)
|
||||
|
||||
func (s *Server) ListFailedEvents(ctx context.Context, _ *system_pb.ListFailedEventsRequest) (*system_pb.ListFailedEventsResponse, error) {
|
||||
failedEventsOld, err := s.administrator.GetFailedEvents(ctx, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
convertedOld := FailedEventsViewToPb(failedEventsOld)
|
||||
|
||||
failedEvents, err := s.query.SearchFailedEvents(ctx, new(query.FailedEventSearchQueries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
convertedNew := FailedEventsToPb(s.database, failedEvents)
|
||||
return &system_pb.ListFailedEventsResponse{Result: append(convertedOld, convertedNew...)}, nil
|
||||
return &system_pb.ListFailedEventsResponse{Result: FailedEventsToPb(s.database, failedEvents)}, nil
|
||||
}
|
||||
|
||||
func (s *Server) RemoveFailedEvent(ctx context.Context, req *system_pb.RemoveFailedEventRequest) (*system_pb.RemoveFailedEventResponse, error) {
|
||||
var err error
|
||||
if req.Database != s.database {
|
||||
err = s.administrator.RemoveFailedEvent(ctx, RemoveFailedEventRequestToModel(req))
|
||||
} else {
|
||||
err = s.query.RemoveFailedEvent(ctx, req.ViewName, req.InstanceId, req.FailedSequence)
|
||||
}
|
||||
err := s.query.RemoveFailedEvent(ctx, req.ViewName, req.InstanceId, req.FailedSequence)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -4,33 +4,9 @@ import (
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/query"
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
|
||||
)
|
||||
|
||||
func FailedEventsViewToPb(failedEvents []*model.FailedEvent) []*system_pb.FailedEvent {
|
||||
events := make([]*system_pb.FailedEvent, len(failedEvents))
|
||||
for i, failedEvent := range failedEvents {
|
||||
events[i] = FailedEventViewToPb(failedEvent)
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
func FailedEventViewToPb(failedEvent *model.FailedEvent) *system_pb.FailedEvent {
|
||||
var lastFailed *timestamppb.Timestamp
|
||||
if !failedEvent.LastFailed.IsZero() {
|
||||
lastFailed = timestamppb.New(failedEvent.LastFailed)
|
||||
}
|
||||
return &system_pb.FailedEvent{
|
||||
Database: failedEvent.Database,
|
||||
ViewName: failedEvent.ViewName,
|
||||
FailedSequence: failedEvent.FailedSequence,
|
||||
FailureCount: failedEvent.FailureCount,
|
||||
ErrorMessage: failedEvent.ErrMsg,
|
||||
LastFailed: lastFailed,
|
||||
}
|
||||
}
|
||||
|
||||
func FailedEventsToPb(database string, failedEvents *query.FailedEvents) []*system_pb.FailedEvent {
|
||||
events := make([]*system_pb.FailedEvent, len(failedEvents.FailedEvents))
|
||||
for i, failedEvent := range failedEvents.FailedEvents {
|
||||
@@ -53,12 +29,3 @@ func FailedEventToPb(database string, failedEvent *query.FailedEvent) *system_pb
|
||||
LastFailed: lastFailed,
|
||||
}
|
||||
}
|
||||
|
||||
func RemoveFailedEventRequestToModel(req *system_pb.RemoveFailedEventRequest) *model.FailedEvent {
|
||||
return &model.FailedEvent{
|
||||
Database: req.Database,
|
||||
ViewName: req.ViewName,
|
||||
FailedSequence: req.FailedSequence,
|
||||
InstanceID: req.InstanceId,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package system_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
system_grpc "github.com/zitadel/zitadel/internal/api/grpc/system"
|
||||
"github.com/zitadel/zitadel/internal/test"
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
|
||||
)
|
||||
|
||||
func TestFailedEventsToPbFields(t *testing.T) {
|
||||
type args struct {
|
||||
failedEvents []*model.FailedEvent
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
}{
|
||||
{
|
||||
name: "all fields",
|
||||
args: args{
|
||||
failedEvents: []*model.FailedEvent{
|
||||
{
|
||||
Database: "admin",
|
||||
ViewName: "users",
|
||||
FailedSequence: 456,
|
||||
FailureCount: 5,
|
||||
LastFailed: time.Now(),
|
||||
ErrMsg: "some error",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := system_grpc.FailedEventsViewToPb(tt.args.failedEvents)
|
||||
for _, g := range got {
|
||||
test.AssertFieldsMapped(t, g)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailedEventToPbFields(t *testing.T) {
|
||||
type args struct {
|
||||
failedEvent *model.FailedEvent
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
}{
|
||||
{
|
||||
"all fields",
|
||||
args{
|
||||
failedEvent: &model.FailedEvent{
|
||||
Database: "admin",
|
||||
ViewName: "users",
|
||||
FailedSequence: 456,
|
||||
FailureCount: 5,
|
||||
LastFailed: time.Now(),
|
||||
ErrMsg: "some error",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
converted := system_grpc.FailedEventViewToPb(tt.args.failedEvent)
|
||||
test.AssertFieldsMapped(t, converted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveFailedEventRequestToModelFields(t *testing.T) {
|
||||
type args struct {
|
||||
req *system_pb.RemoveFailedEventRequest
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
}{
|
||||
{
|
||||
"all fields",
|
||||
args{
|
||||
req: &system_pb.RemoveFailedEventRequest{
|
||||
Database: "admin",
|
||||
ViewName: "users",
|
||||
FailedSequence: 456,
|
||||
InstanceId: "instanceID",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
converted := system_grpc.RemoveFailedEventRequestToModel(tt.args.req)
|
||||
test.AssertFieldsMapped(t, converted, "FailureCount", "LastFailed", "ErrMsg")
|
||||
}
|
||||
}
|
||||
@@ -103,7 +103,7 @@ func (s *Server) ListIAMMembers(ctx context.Context, req *system_pb.ListIAMMembe
|
||||
return nil, err
|
||||
}
|
||||
return &system_pb.ListIAMMembersResponse{
|
||||
Details: object.ToListDetails(res.Count, res.Sequence, res.Timestamp),
|
||||
Details: object.ToListDetails(res.Count, res.Sequence, res.LastRun),
|
||||
//TODO: resource owner of user of the member instead of the membership resource owner
|
||||
Result: member.MembersToPb("", res.Members),
|
||||
}, nil
|
||||
@@ -146,7 +146,7 @@ func (s *Server) ListDomains(ctx context.Context, req *system_pb.ListDomainsRequ
|
||||
}
|
||||
return &system_pb.ListDomainsResponse{
|
||||
Result: instance_grpc.DomainsToPb(domains.Domains),
|
||||
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.Timestamp),
|
||||
Details: object.ToListDetails(domains.Count, domains.Sequence, domains.LastRun),
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@ package system
|
||||
import (
|
||||
"google.golang.org/grpc"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/admin/repository"
|
||||
"github.com/zitadel/zitadel/internal/admin/repository/eventsourcing"
|
||||
"github.com/zitadel/zitadel/internal/api/authz"
|
||||
"github.com/zitadel/zitadel/internal/api/grpc/server"
|
||||
@@ -23,7 +22,6 @@ type Server struct {
|
||||
database string
|
||||
command *command.Commands
|
||||
query *query.Queries
|
||||
administrator repository.AdministratorRepository
|
||||
defaultInstance command.InstanceSetup
|
||||
externalDomain string
|
||||
}
|
||||
@@ -35,7 +33,6 @@ type Config struct {
|
||||
func CreateServer(
|
||||
command *command.Commands,
|
||||
query *query.Queries,
|
||||
repo repository.Repository,
|
||||
database string,
|
||||
defaultInstance command.InstanceSetup,
|
||||
externalDomain string,
|
||||
@@ -43,7 +40,6 @@ func CreateServer(
|
||||
return &Server{
|
||||
command: command,
|
||||
query: query,
|
||||
administrator: repo,
|
||||
database: database,
|
||||
defaultInstance: defaultInstance,
|
||||
externalDomain: externalDomain,
|
||||
|
||||
@@ -8,28 +8,15 @@ import (
|
||||
)
|
||||
|
||||
func (s *Server) ListViews(ctx context.Context, _ *system_pb.ListViewsRequest) (*system_pb.ListViewsResponse, error) {
|
||||
currentSequences, err := s.query.SearchCurrentSequences(ctx, new(query.CurrentSequencesSearchQueries))
|
||||
currentSequences, err := s.query.SearchCurrentStates(ctx, new(query.CurrentStateSearchQueries))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
convertedCurrentSequences := CurrentSequencesToPb(s.database, currentSequences)
|
||||
views, err := s.administrator.GetViews("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
convertedViews := ViewsToPb(views)
|
||||
|
||||
convertedCurrentSequences = append(convertedCurrentSequences, convertedViews...)
|
||||
return &system_pb.ListViewsResponse{Result: convertedCurrentSequences}, nil
|
||||
return &system_pb.ListViewsResponse{Result: CurrentSequencesToPb(s.database, currentSequences)}, nil
|
||||
}
|
||||
|
||||
func (s *Server) ClearView(ctx context.Context, req *system_pb.ClearViewRequest) (*system_pb.ClearViewResponse, error) {
|
||||
var err error
|
||||
if req.Database != s.database {
|
||||
err = s.administrator.ClearView(ctx, req.Database, req.ViewName)
|
||||
} else {
|
||||
err = s.query.ClearCurrentSequence(ctx, req.ViewName)
|
||||
}
|
||||
err := s.query.ClearCurrentSequence(ctx, req.ViewName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -4,41 +4,22 @@ import (
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
"github.com/zitadel/zitadel/internal/query"
|
||||
"github.com/zitadel/zitadel/internal/view/model"
|
||||
system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
|
||||
)
|
||||
|
||||
func ViewsToPb(views []*model.View) []*system_pb.View {
|
||||
v := make([]*system_pb.View, len(views))
|
||||
for i, view := range views {
|
||||
v[i] = ViewToPb(view)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func ViewToPb(view *model.View) *system_pb.View {
|
||||
return &system_pb.View{
|
||||
Database: view.Database,
|
||||
ViewName: view.ViewName,
|
||||
LastSuccessfulSpoolerRun: timestamppb.New(view.LastSuccessfulSpoolerRun),
|
||||
ProcessedSequence: view.CurrentSequence,
|
||||
EventTimestamp: timestamppb.New(view.EventTimestamp),
|
||||
}
|
||||
}
|
||||
|
||||
func CurrentSequencesToPb(database string, currentSequences *query.CurrentSequences) []*system_pb.View {
|
||||
v := make([]*system_pb.View, len(currentSequences.CurrentSequences))
|
||||
for i, currentSequence := range currentSequences.CurrentSequences {
|
||||
func CurrentSequencesToPb(database string, currentSequences *query.CurrentStates) []*system_pb.View {
|
||||
v := make([]*system_pb.View, len(currentSequences.CurrentStates))
|
||||
for i, currentSequence := range currentSequences.CurrentStates {
|
||||
v[i] = CurrentSequenceToPb(database, currentSequence)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func CurrentSequenceToPb(database string, currentSequence *query.CurrentSequence) *system_pb.View {
|
||||
func CurrentSequenceToPb(database string, currentSequence *query.CurrentState) *system_pb.View {
|
||||
return &system_pb.View{
|
||||
Database: database,
|
||||
ViewName: currentSequence.ProjectionName,
|
||||
ProcessedSequence: currentSequence.CurrentSequence,
|
||||
LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.Timestamp),
|
||||
ProcessedSequence: currentSequence.Sequence,
|
||||
LastSuccessfulSpoolerRun: timestamppb.New(currentSequence.LastRun),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user