feat: System api (#3461)

* feat: start system api

* feat: remove auth

* feat: change gitignore

* feat: run system api

* feat: remove clear view form admin api

* feat: search instances

* feat: add instance

* fix: set primary domain

* Update .gitignore

* fix: add instance

* fix: add instance

* fix: handle errors

* fix: handle instance name

* fix: test

Co-authored-by: Livio Amstutz <[email protected]>

internal/api/grpc/system/failed_event.go

@@ -0,0 +1,37 @@
+package system
+
+import (
+	"context"
+
+	"github.com/caos/zitadel/internal/query"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func (s *Server) ListFailedEvents(ctx context.Context, req *system_pb.ListFailedEventsRequest) (*system_pb.ListFailedEventsResponse, error) {
+	failedEventsOld, err := s.administrator.GetFailedEvents(ctx)
+	if err != nil {
+		return nil, err
+	}
+	convertedOld := FailedEventsViewToPb(failedEventsOld)
+
+	failedEvents, err := s.query.SearchFailedEvents(ctx, new(query.FailedEventSearchQueries))
+	if err != nil {
+		return nil, err
+	}
+	convertedNew := FailedEventsToPb(failedEvents)
+	convertedOld = append(convertedOld, convertedNew...)
+	return &system_pb.ListFailedEventsResponse{Result: convertedOld}, nil
+}
+
+func (s *Server) RemoveFailedEvent(ctx context.Context, req *system_pb.RemoveFailedEventRequest) (*system_pb.RemoveFailedEventResponse, error) {
+	var err error
+	if req.Database != "zitadel" {
+		err = s.administrator.RemoveFailedEvent(ctx, RemoveFailedEventRequestToModel(req))
+	} else {
+		err = s.query.RemoveFailedEvent(ctx, req.ViewName, req.FailedSequence)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.RemoveFailedEventResponse{}, nil
+}

internal/api/grpc/system/failed_event_converter.go

@@ -0,0 +1,51 @@
+package system
+
+import (
+	"github.com/caos/zitadel/internal/query"
+	"github.com/caos/zitadel/internal/view/model"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func FailedEventsViewToPb(failedEvents []*model.FailedEvent) []*system_pb.FailedEvent {
+	events := make([]*system_pb.FailedEvent, len(failedEvents))
+	for i, failedEvent := range failedEvents {
+		events[i] = FailedEventViewToPb(failedEvent)
+	}
+	return events
+}
+
+func FailedEventViewToPb(failedEvent *model.FailedEvent) *system_pb.FailedEvent {
+	return &system_pb.FailedEvent{
+		Database:       failedEvent.Database,
+		ViewName:       failedEvent.ViewName,
+		FailedSequence: failedEvent.FailedSequence,
+		FailureCount:   failedEvent.FailureCount,
+		ErrorMessage:   failedEvent.ErrMsg,
+	}
+}
+
+func FailedEventsToPb(failedEvents *query.FailedEvents) []*system_pb.FailedEvent {
+	events := make([]*system_pb.FailedEvent, len(failedEvents.FailedEvents))
+	for i, failedEvent := range failedEvents.FailedEvents {
+		events[i] = FailedEventToPb(failedEvent)
+	}
+	return events
+}
+
+func FailedEventToPb(failedEvent *query.FailedEvent) *system_pb.FailedEvent {
+	return &system_pb.FailedEvent{
+		Database:       "zitadel",
+		ViewName:       failedEvent.ProjectionName,
+		FailedSequence: failedEvent.FailedSequence,
+		FailureCount:   failedEvent.FailureCount,
+		ErrorMessage:   failedEvent.Error,
+	}
+}
+
+func RemoveFailedEventRequestToModel(req *system_pb.RemoveFailedEventRequest) *model.FailedEvent {
+	return &model.FailedEvent{
+		Database:       req.Database,
+		ViewName:       req.ViewName,
+		FailedSequence: req.FailedSequence,
+	}
+}

internal/api/grpc/system/failed_event_converter_test.go

@@ -0,0 +1,95 @@
+package system_test
+
+import (
+	"testing"
+
+	system_grpc "github.com/caos/zitadel/internal/api/grpc/system"
+	"github.com/caos/zitadel/internal/test"
+	"github.com/caos/zitadel/internal/view/model"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func TestFailedEventsToPbFields(t *testing.T) {
+	type args struct {
+		failedEvents []*model.FailedEvent
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "all fields",
+			args: args{
+				failedEvents: []*model.FailedEvent{
+					{
+						Database:       "admin",
+						ViewName:       "users",
+						FailedSequence: 456,
+						FailureCount:   5,
+						ErrMsg:         "some error",
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := system_grpc.FailedEventsViewToPb(tt.args.failedEvents)
+			for _, g := range got {
+				test.AssertFieldsMapped(t, g)
+			}
+		})
+	}
+}
+
+func TestFailedEventToPbFields(t *testing.T) {
+	type args struct {
+		failedEvent *model.FailedEvent
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			"all fields",
+			args{
+				failedEvent: &model.FailedEvent{
+					Database:       "admin",
+					ViewName:       "users",
+					FailedSequence: 456,
+					FailureCount:   5,
+					ErrMsg:         "some error",
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		converted := system_grpc.FailedEventViewToPb(tt.args.failedEvent)
+		test.AssertFieldsMapped(t, converted)
+	}
+}
+
+func TestRemoveFailedEventRequestToModelFields(t *testing.T) {
+	type args struct {
+		req *system_pb.RemoveFailedEventRequest
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			"all fields",
+			args{
+				req: &system_pb.RemoveFailedEventRequest{
+					Database:       "admin",
+					ViewName:       "users",
+					FailedSequence: 456,
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		converted := system_grpc.RemoveFailedEventRequestToModel(tt.args.req)
+		test.AssertFieldsMapped(t, converted, "FailureCount", "ErrMsg")
+	}
+}

internal/api/grpc/system/instance.go

@@ -0,0 +1,127 @@
+package system
+
+import (
+	"context"
+
+	"github.com/caos/zitadel/internal/api/authz"
+	instance_grpc "github.com/caos/zitadel/internal/api/grpc/instance"
+	"github.com/caos/zitadel/internal/api/grpc/object"
+	object_pb "github.com/caos/zitadel/pkg/grpc/object"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func (s *Server) ListInstances(ctx context.Context, req *system_pb.ListInstancesRequest) (*system_pb.ListInstancesResponse, error) {
+	queries, err := ListInstancesRequestToModel(req)
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := s.query.SearchInstances(ctx, queries)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.ListInstancesResponse{
+		Result: instance_grpc.InstancesToPb(result.Instances),
+		Details: &object_pb.ListDetails{
+			TotalResult: result.Count,
+		},
+	}, nil
+}
+
+func (s *Server) GetInstance(ctx context.Context, req *system_pb.GetInstanceRequest) (*system_pb.GetInstanceResponse, error) {
+	ctx = authz.WithInstanceID(ctx, req.Id)
+	instance, err := s.query.Instance(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.GetInstanceResponse{
+		Instance: instance_grpc.InstanceToPb(instance),
+	}, nil
+}
+
+func (s *Server) AddInstance(ctx context.Context, req *system_pb.AddInstanceRequest) (*system_pb.AddInstanceResponse, error) {
+	id, details, err := s.command.SetUpInstance(ctx, AddInstancePbToSetupInstance(req, s.DefaultInstance), s.ExternalSecure, s.BaseURL)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.AddInstanceResponse{
+		Id: id,
+		Details: object.AddToDetailsPb(
+			details.Sequence,
+			details.EventDate,
+			details.ResourceOwner,
+		),
+	}, nil
+	return nil, nil
+}
+
+func (s *Server) ListDomains(ctx context.Context, req *system_pb.ListDomainsRequest) (*system_pb.ListDomainsResponse, error) {
+	ctx = authz.WithInstanceID(ctx, req.Id)
+	queries, err := ListInstanceDomainsRequestToModel(req)
+	if err != nil {
+		return nil, err
+	}
+
+	domains, err := s.query.SearchInstanceDomains(ctx, queries)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.ListDomainsResponse{
+		Result: instance_grpc.DomainsToPb(domains.Domains),
+		Details: object.ToListDetails(
+			domains.Count,
+			domains.Sequence,
+			domains.Timestamp,
+		),
+	}, nil
+}
+
+func (s *Server) AddDomain(ctx context.Context, req *system_pb.AddDomainRequest) (*system_pb.AddDomainResponse, error) {
+	ctx = authz.WithInstanceID(ctx, req.Id)
+	instance, err := s.query.Instance(ctx)
+	if err != nil {
+		return nil, err
+	}
+	ctx = authz.WithInstance(ctx, instance)
+	details, err := s.command.AddInstanceDomain(ctx, req.Domain)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.AddDomainResponse{
+		Details: object.AddToDetailsPb(
+			details.Sequence,
+			details.EventDate,
+			details.ResourceOwner,
+		),
+	}, nil
+}
+
+func (s *Server) RemoveDomain(ctx context.Context, req *system_pb.RemoveDomainRequest) (*system_pb.RemoveDomainResponse, error) {
+	ctx = authz.WithInstanceID(ctx, req.Id)
+	details, err := s.command.RemoveInstanceDomain(ctx, req.Domain)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.RemoveDomainResponse{
+		Details: object.ChangeToDetailsPb(
+			details.Sequence,
+			details.EventDate,
+			details.ResourceOwner,
+		),
+	}, nil
+}
+
+func (s *Server) SetPrimaryDomain(ctx context.Context, req *system_pb.SetPrimaryDomainRequest) (*system_pb.SetPrimaryDomainResponse, error) {
+	ctx = authz.WithInstanceID(ctx, req.Id)
+	details, err := s.command.SetPrimaryInstanceDomain(ctx, req.Domain)
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.SetPrimaryDomainResponse{
+		Details: object.ChangeToDetailsPb(
+			details.Sequence,
+			details.EventDate,
+			details.ResourceOwner,
+		),
+	}, nil
+}

internal/api/grpc/system/instance_converter.go

@@ -0,0 +1,97 @@
+package system
+
+import (
+	instance_grpc "github.com/caos/zitadel/internal/api/grpc/instance"
+	"github.com/caos/zitadel/internal/api/grpc/object"
+	"github.com/caos/zitadel/internal/command"
+	"github.com/caos/zitadel/internal/query"
+	instance_pb "github.com/caos/zitadel/pkg/grpc/instance"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func AddInstancePbToSetupInstance(req *system_pb.AddInstanceRequest, defaultInstance command.InstanceSetup) *command.InstanceSetup {
+	if req.InstanceName != "" {
+		defaultInstance.InstanceName = req.InstanceName
+		defaultInstance.Org.Name = req.InstanceName
+	}
+	if req.CustomDomain != "" {
+		defaultInstance.CustomDomain = req.CustomDomain
+	}
+	if req.FirstOrgName != "" {
+		defaultInstance.Org.Name = req.FirstOrgName
+	}
+	if req.OwnerEmail != "" {
+		defaultInstance.Org.Human.Email.Address = req.OwnerEmail
+	}
+	if req.OwnerUsername != "" {
+		defaultInstance.Org.Human.Username = req.OwnerUsername
+	}
+	if req.OwnerFirstName != "" {
+		defaultInstance.Org.Human.FirstName = req.OwnerFirstName
+	}
+	if req.OwnerLastName != "" {
+		defaultInstance.Org.Human.LastName = req.OwnerLastName
+	}
+	return &defaultInstance
+}
+func ListInstancesRequestToModel(req *system_pb.ListInstancesRequest) (*query.InstanceSearchQueries, error) {
+	offset, limit, asc := object.ListQueryToModel(req.Query)
+	queries, err := instance_grpc.InstanceQueriesToModel(req.Queries)
+	if err != nil {
+		return nil, err
+	}
+	return &query.InstanceSearchQueries{
+		SearchRequest: query.SearchRequest{
+			Offset:        offset,
+			Limit:         limit,
+			Asc:           asc,
+			SortingColumn: fieldNameToInstanceColumn(req.SortingColumn),
+		},
+		Queries: queries,
+	}, nil
+}
+
+func fieldNameToInstanceColumn(fieldName instance_pb.FieldName) query.Column {
+	switch fieldName {
+	case instance_pb.FieldName_FIELD_NAME_ID:
+		return query.InstanceColumnID
+	case instance_pb.FieldName_FIELD_NAME_NAME:
+		return query.InstanceColumnName
+	case instance_pb.FieldName_FIELD_NAME_CREATION_DATE:
+		return query.InstanceColumnCreationDate
+	default:
+		return query.Column{}
+	}
+}
+
+func ListInstanceDomainsRequestToModel(req *system_pb.ListDomainsRequest) (*query.InstanceDomainSearchQueries, error) {
+	offset, limit, asc := object.ListQueryToModel(req.Query)
+	queries, err := instance_grpc.DomainQueriesToModel(req.Queries)
+	if err != nil {
+		return nil, err
+	}
+	return &query.InstanceDomainSearchQueries{
+		SearchRequest: query.SearchRequest{
+			Offset:        offset,
+			Limit:         limit,
+			Asc:           asc,
+			SortingColumn: fieldNameToInstanceDomainColumn(req.SortingColumn),
+		},
+		Queries: queries,
+	}, nil
+}
+
+func fieldNameToInstanceDomainColumn(fieldName instance_pb.DomainFieldName) query.Column {
+	switch fieldName {
+	case instance_pb.DomainFieldName_DOMAIN_FIELD_NAME_DOMAIN:
+		return query.InstanceDomainDomainCol
+	case instance_pb.DomainFieldName_DOMAIN_FIELD_NAME_GENERATED:
+		return query.InstanceDomainIsGeneratedCol
+	case instance_pb.DomainFieldName_DOMAIN_FIELD_NAME_PRIMARY:
+		return query.InstanceDomainIsPrimaryCol
+	case instance_pb.DomainFieldName_DOMAIN_FIELD_NAME_CREATION_DATE:
+		return query.InstanceDomainCreationDateCol
+	default:
+		return query.Column{}
+	}
+}

internal/api/grpc/system/server.go

@@ -0,0 +1,75 @@
+package system
+
+import (
+	"github.com/caos/zitadel/internal/admin/repository"
+	http_util "github.com/caos/zitadel/internal/api/http"
+	"google.golang.org/grpc"
+
+	"github.com/caos/zitadel/internal/admin/repository/eventsourcing"
+	"github.com/caos/zitadel/internal/api/authz"
+	"github.com/caos/zitadel/internal/api/grpc/server"
+	"github.com/caos/zitadel/internal/command"
+	"github.com/caos/zitadel/internal/query"
+	"github.com/caos/zitadel/pkg/grpc/system"
+)
+
+const (
+	systemAPI = "System-API"
+)
+
+var _ system.SystemServiceServer = (*Server)(nil)
+
+type Server struct {
+	system.UnimplementedSystemServiceServer
+	command         *command.Commands
+	query           *query.Queries
+	administrator   repository.AdministratorRepository
+	DefaultInstance command.InstanceSetup
+	ExternalSecure  bool
+	BaseURL         string
+}
+
+type Config struct {
+	Repository eventsourcing.Config
+}
+
+func CreateServer(command *command.Commands,
+	query *query.Queries,
+	repo repository.Repository,
+	defaultInstance command.InstanceSetup,
+	externalPort uint16,
+	externalDomain string,
+	externalSecure bool) *Server {
+	return &Server{
+		command:         command,
+		query:           query,
+		administrator:   repo,
+		DefaultInstance: defaultInstance,
+		ExternalSecure:  externalSecure,
+		BaseURL:         http_util.BuildHTTP(externalDomain, externalPort, externalSecure),
+	}
+}
+
+func (s *Server) RegisterServer(grpcServer *grpc.Server) {
+	system.RegisterSystemServiceServer(grpcServer, s)
+}
+
+func (s *Server) AppName() string {
+	return systemAPI
+}
+
+func (s *Server) MethodPrefix() string {
+	return system.SystemService_MethodPrefix
+}
+
+func (s *Server) AuthMethods() authz.MethodMapping {
+	return system.SystemService_AuthMethods
+}
+
+func (s *Server) RegisterGateway() server.GatewayFunc {
+	return system.RegisterSystemServiceHandlerFromEndpoint
+}
+
+func (s *Server) GatewayPathPrefix() string {
+	return "/system/v1"
+}

internal/api/grpc/system/view.go

@@ -0,0 +1,37 @@
+package system
+
+import (
+	"context"
+
+	"github.com/caos/zitadel/internal/query"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+)
+
+func (s *Server) ListViews(ctx context.Context, _ *system_pb.ListViewsRequest) (*system_pb.ListViewsResponse, error) {
+	currentSequences, err := s.query.SearchCurrentSequences(ctx, new(query.CurrentSequencesSearchQueries))
+	if err != nil {
+		return nil, err
+	}
+	convertedCurrentSequences := CurrentSequencesToPb(currentSequences)
+	views, err := s.administrator.GetViews()
+	if err != nil {
+		return nil, err
+	}
+	convertedViews := ViewsToPb(views)
+
+	convertedCurrentSequences = append(convertedCurrentSequences, convertedViews...)
+	return &system_pb.ListViewsResponse{Result: convertedCurrentSequences}, nil
+}
+
+func (s *Server) ClearView(ctx context.Context, req *system_pb.ClearViewRequest) (*system_pb.ClearViewResponse, error) {
+	var err error
+	if req.Database != "zitadel" {
+		err = s.administrator.ClearView(ctx, req.Database, req.ViewName)
+	} else {
+		err = s.query.ClearCurrentSequence(ctx, req.ViewName)
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &system_pb.ClearViewResponse{}, nil
+}

internal/api/grpc/system/view_converter.go

@@ -0,0 +1,43 @@
+package system
+
+import (
+	"github.com/caos/zitadel/internal/query"
+	"github.com/caos/zitadel/internal/view/model"
+	system_pb "github.com/caos/zitadel/pkg/grpc/system"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func ViewsToPb(views []*model.View) []*system_pb.View {
+	v := make([]*system_pb.View, len(views))
+	for i, view := range views {
+		v[i] = ViewToPb(view)
+	}
+	return v
+}
+
+func ViewToPb(view *model.View) *system_pb.View {
+	return &system_pb.View{
+		Database:                 view.Database,
+		ViewName:                 view.ViewName,
+		LastSuccessfulSpoolerRun: timestamppb.New(view.LastSuccessfulSpoolerRun),
+		ProcessedSequence:        view.CurrentSequence,
+		EventTimestamp:           timestamppb.New(view.EventTimestamp),
+	}
+}
+
+func CurrentSequencesToPb(currentSequences *query.CurrentSequences) []*system_pb.View {
+	v := make([]*system_pb.View, len(currentSequences.CurrentSequences))
+	for i, currentSequence := range currentSequences.CurrentSequences {
+		v[i] = CurrentSequenceToPb(currentSequence)
+	}
+	return v
+}
+
+func CurrentSequenceToPb(currentSequence *query.CurrentSequence) *system_pb.View {
+	return &system_pb.View{
+		Database:          "zitadel",
+		ViewName:          currentSequence.ProjectionName,
+		ProcessedSequence: currentSequence.CurrentSequence,
+		EventTimestamp:    timestamppb.New(currentSequence.Timestamp),
+	}
+}