fix(spooler): correct workers (#508)

* one concurrent task

* disable spooler

* fix: improve concurrency in spooler

* fix: dont block lock

* fix: break if lock failed

* fix: check if handler is working

* fix: worker id

* fix: test

* fix: use limit for spoolers configured in startup.yaml

* fix test

* fix: factory

* fix(key): only reduce if not expired

* fix(searchQueryFactory): check for string-slice in aggregateID

* fix(migrations): combine migrations

* fix: allow saving multiple objects in one request

* fix(eventstore): logging

* fix(eventstore): rethink insert i locks table

* fix: ignore failed tests for the moment

* fix: tuubel

* fix: for tests in io

* fix: ignore tests for io

* fix: rename concurrent tasks to workers

* fix: incomment tests and remove some tests

* fix: refert changes for io

* refactor(eventstore): combine types of sql in one file

* refactor(eventstore): logs, TODO's, tests

* fix(eventstore): sql package

* test(eventstore): add tests for search query factory

* chore: logs

* fix(spooler): optimize lock query
chore(migrations): rename locks.object_type to view_name
chore(migrations): refactor migrations

* test: incomment tests

* fix: rename PrepareSaves to PrepareBulkSave

* chore: go dependencies

* fix(migrations): add id in events table

* refactor(lock): less magic numbers

Co-authored-by: Livio Amstutz <[email protected]>

internal/eventstore/internal/repository/sql/db_mock_test.go

@@ -2,26 +2,24 @@ package sql
 
 import (
 	"database/sql"
-	"fmt"
 	"regexp"
 	"testing"
 	"time"
 
 	"github.com/DATA-DOG/go-sqlmock"
 	"github.com/caos/zitadel/internal/eventstore/models"
-	"github.com/lib/pq"
 )
 
 const (
-	selectEscaped = `SELECT id, creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore\.events`
+	selectEscaped = `SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore\.events WHERE aggregate_type = \$1`
 )
 
 var (
-	eventColumns                             = []string{"id", "creation_date", "event_type", "event_sequence", "previous_sequence", "event_data", "editor_service", "editor_user", "resource_owner", "aggregate_type", "aggregate_id", "aggregate_version"}
-	expectedFilterEventsLimitFormat          = regexp.MustCompile(selectEscaped + ` ORDER BY event_sequence LIMIT \$1`).String()
+	eventColumns                             = []string{"creation_date", "event_type", "event_sequence", "previous_sequence", "event_data", "editor_service", "editor_user", "resource_owner", "aggregate_type", "aggregate_id", "aggregate_version"}
+	expectedFilterEventsLimitFormat          = regexp.MustCompile(selectEscaped + ` ORDER BY event_sequence LIMIT \$2`).String()
 	expectedFilterEventsDescFormat           = regexp.MustCompile(selectEscaped + ` ORDER BY event_sequence DESC`).String()
-	expectedFilterEventsAggregateIDLimit     = regexp.MustCompile(selectEscaped + ` WHERE aggregate_id = \$1 ORDER BY event_sequence LIMIT \$2`).String()
-	expectedFilterEventsAggregateIDTypeLimit = regexp.MustCompile(selectEscaped + ` WHERE aggregate_id = \$1 AND aggregate_type = ANY\(\$2\) ORDER BY event_sequence LIMIT \$3`).String()
+	expectedFilterEventsAggregateIDLimit     = regexp.MustCompile(selectEscaped + ` AND aggregate_id = \$2 ORDER BY event_sequence LIMIT \$3`).String()
+	expectedFilterEventsAggregateIDTypeLimit = regexp.MustCompile(selectEscaped + ` AND aggregate_id = \$2 ORDER BY event_sequence LIMIT \$3`).String()
 	expectedGetAllEvents                     = regexp.MustCompile(selectEscaped + ` ORDER BY event_sequence`).String()
 
 	expectedInsertStatement = regexp.MustCompile(`INSERT INTO eventstore\.events ` +
@@ -29,7 +27,7 @@ var (
 		`SELECT \$1, \$2, \$3, \$4, COALESCE\(\$5, now\(\)\), \$6, \$7, \$8, \$9, \$10 ` +
 		`WHERE EXISTS \(` +
 		`SELECT 1 FROM eventstore\.events WHERE aggregate_type = \$11 AND aggregate_id = \$12 HAVING MAX\(event_sequence\) = \$13 OR \(\$14::BIGINT IS NULL AND COUNT\(\*\) = 0\)\) ` +
-		`RETURNING id, event_sequence, creation_date`).String()
+		`RETURNING event_sequence, creation_date`).String()
 )
 
 type dbMock struct {
@@ -98,15 +96,15 @@ func (db *dbMock) expectRollback(err error) *dbMock {
 	return db
 }
 
-func (db *dbMock) expectInsertEvent(e *models.Event, returnedID string, returnedSequence uint64) *dbMock {
+func (db *dbMock) expectInsertEvent(e *models.Event, returnedSequence uint64) *dbMock {
 	db.mock.ExpectQuery(expectedInsertStatement).
 		WithArgs(
 			e.Type, e.AggregateType, e.AggregateID, e.AggregateVersion, sqlmock.AnyArg(), Data(e.Data), e.EditorUser, e.EditorService, e.ResourceOwner, Sequence(e.PreviousSequence),
 			e.AggregateType, e.AggregateID, Sequence(e.PreviousSequence), Sequence(e.PreviousSequence),
 		).
 		WillReturnRows(
-			sqlmock.NewRows([]string{"id", "event_sequence", "creation_date"}).
-				AddRow(returnedID, returnedSequence, time.Now().UTC()),
+			sqlmock.NewRows([]string{"event_sequence", "creation_date"}).
+				AddRow(returnedSequence, time.Now().UTC()),
 		)
 
 	return db
@@ -123,45 +121,45 @@ func (db *dbMock) expectInsertEventError(e *models.Event) *dbMock {
 	return db
 }
 
-func (db *dbMock) expectFilterEventsLimit(limit uint64, eventCount int) *dbMock {
+func (db *dbMock) expectFilterEventsLimit(aggregateType string, limit uint64, eventCount int) *dbMock {
 	rows := sqlmock.NewRows(eventColumns)
 	for i := 0; i < eventCount; i++ {
-		rows.AddRow(fmt.Sprint("event", i), time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
+		rows.AddRow(time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
 	}
 	db.mock.ExpectQuery(expectedFilterEventsLimitFormat).
-		WithArgs(limit).
+		WithArgs(aggregateType, limit).
 		WillReturnRows(rows)
 	return db
 }
 
-func (db *dbMock) expectFilterEventsDesc(eventCount int) *dbMock {
+func (db *dbMock) expectFilterEventsDesc(aggregateType string, eventCount int) *dbMock {
 	rows := sqlmock.NewRows(eventColumns)
 	for i := eventCount; i > 0; i-- {
-		rows.AddRow(fmt.Sprint("event", i), time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
+		rows.AddRow(time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
 	}
 	db.mock.ExpectQuery(expectedFilterEventsDescFormat).
 		WillReturnRows(rows)
 	return db
 }
 
-func (db *dbMock) expectFilterEventsAggregateIDLimit(aggregateID string, limit uint64) *dbMock {
+func (db *dbMock) expectFilterEventsAggregateIDLimit(aggregateType, aggregateID string, limit uint64) *dbMock {
 	rows := sqlmock.NewRows(eventColumns)
 	for i := limit; i > 0; i-- {
-		rows.AddRow(fmt.Sprint("event", i), time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
+		rows.AddRow(time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
 	}
 	db.mock.ExpectQuery(expectedFilterEventsAggregateIDLimit).
-		WithArgs(aggregateID, limit).
+		WithArgs(aggregateType, aggregateID, limit).
 		WillReturnRows(rows)
 	return db
 }
 
-func (db *dbMock) expectFilterEventsAggregateIDTypeLimit(aggregateID, aggregateType string, limit uint64) *dbMock {
+func (db *dbMock) expectFilterEventsAggregateIDTypeLimit(aggregateType, aggregateID string, limit uint64) *dbMock {
 	rows := sqlmock.NewRows(eventColumns)
 	for i := limit; i > 0; i-- {
-		rows.AddRow(fmt.Sprint("event", i), time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
+		rows.AddRow(time.Now(), "eventType", Sequence(i+1), Sequence(i), nil, "svc", "hodor", "org", "aggType", "aggID", "v1.0.0")
 	}
 	db.mock.ExpectQuery(expectedFilterEventsAggregateIDTypeLimit).
-		WithArgs(aggregateID, pq.Array([]string{aggregateType}), limit).
+		WithArgs(aggregateType, aggregateID, limit).
 		WillReturnRows(rows)
 	return db
 }
@@ -172,8 +170,24 @@ func (db *dbMock) expectFilterEventsError(returnedErr error) *dbMock {
 	return db
 }
 
-func (db *dbMock) expectPrepareInsert() *dbMock {
-	db.mock.ExpectPrepare(expectedInsertStatement)
+func (db *dbMock) expectLatestSequenceFilter(aggregateType string, sequence Sequence) *dbMock {
+	db.mock.ExpectQuery(`SELECT MAX\(event_sequence\) FROM eventstore\.events WHERE aggregate_type = \$1`).
+		WithArgs(aggregateType).
+		WillReturnRows(sqlmock.NewRows([]string{"max_sequence"}).AddRow(sequence))
+	return db
+}
+
+func (db *dbMock) expectLatestSequenceFilterError(aggregateType string, err error) *dbMock {
+	db.mock.ExpectQuery(`SELECT MAX\(event_sequence\) FROM eventstore\.events WHERE aggregate_type = \$1`).
+		WithArgs(aggregateType).WillReturnError(err)
+	return db
+}
+
+func (db *dbMock) expectPrepareInsert(err error) *dbMock {
+	prepare := db.mock.ExpectPrepare(expectedInsertStatement)
+	if err != nil {
+		prepare.WillReturnError(err)
+	}
 
 	return db
 }

internal/eventstore/internal/repository/sql/filter.go

@@ -3,44 +3,26 @@ package sql
 import (
 	"context"
 	"database/sql"
-	"fmt"
-	"strconv"
-	"strings"
 
 	"github.com/caos/logging"
 	"github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 	es_models "github.com/caos/zitadel/internal/eventstore/models"
-	"github.com/lib/pq"
-)
-
-const (
-	selectStmt = "SELECT" +
-		" id" +
-		", creation_date" +
-		", event_type" +
-		", event_sequence" +
-		", previous_sequence" +
-		", event_data" +
-		", editor_service" +
-		", editor_user" +
-		", resource_owner" +
-		", aggregate_type" +
-		", aggregate_id" +
-		", aggregate_version" +
-		" FROM eventstore.events"
 )
 
 type Querier interface {
 	Query(query string, args ...interface{}) (*sql.Rows, error)
 }
 
-func (db *SQL) Filter(ctx context.Context, searchQuery *es_models.SearchQuery) (events []*models.Event, err error) {
+func (db *SQL) Filter(ctx context.Context, searchQuery *es_models.SearchQueryFactory) (events []*models.Event, err error) {
 	return filter(db.client, searchQuery)
 }
 
-func filter(querier Querier, searchQuery *es_models.SearchQuery) (events []*es_models.Event, err error) {
-	query, values := prepareQuery(searchQuery)
+func filter(querier Querier, searchQuery *es_models.SearchQueryFactory) (events []*es_models.Event, err error) {
+	query, limit, values, rowScanner := buildQuery(searchQuery)
+	if query == "" {
+		return nil, errors.ThrowInvalidArgument(nil, "SQL-rWeBw", "invalid query factory")
+	}
 
 	rows, err := querier.Query(query, values...)
 	if err != nil {
@@ -49,136 +31,32 @@ func filter(querier Querier, searchQuery *es_models.SearchQuery) (events []*es_m
 	}
 	defer rows.Close()
 
-	events = make([]*es_models.Event, 0, searchQuery.Limit)
+	events = make([]*es_models.Event, 0, limit)
 
 	for rows.Next() {
 		event := new(models.Event)
-		var previousSequence Sequence
-		data := make(Data, 0)
-
-		err = rows.Scan(
-			&event.ID,
-			&event.CreationDate,
-			&event.Type,
-			&event.Sequence,
-			&previousSequence,
-			&data,
-			&event.EditorService,
-			&event.EditorUser,
-			&event.ResourceOwner,
-			&event.AggregateType,
-			&event.AggregateID,
-			&event.AggregateVersion,
-		)
-
+		err := rowScanner(rows.Scan, event)
 		if err != nil {
-			logging.Log("SQL-wHNPo").WithError(err).Warn("unable to scan row")
-			return nil, errors.ThrowInternal(err, "SQL-BfZwF", "unable to scan row")
+			return nil, err
 		}
 
-		event.PreviousSequence = uint64(previousSequence)
-
-		event.Data = make([]byte, len(data))
-		copy(event.Data, data)
-
 		events = append(events, event)
 	}
 
 	return events, nil
 }
 
-func prepareQuery(searchQuery *es_models.SearchQuery) (query string, values []interface{}) {
-	where, values := prepareWhere(searchQuery)
-	query = selectStmt + where
-
-	query += " ORDER BY event_sequence"
-	if searchQuery.Desc {
-		query += " DESC"
+func (db *SQL) LatestSequence(ctx context.Context, queryFactory *es_models.SearchQueryFactory) (uint64, error) {
+	query, _, values, rowScanner := buildQuery(queryFactory)
+	if query == "" {
+		return 0, errors.ThrowInvalidArgument(nil, "SQL-rWeBw", "invalid query factory")
 	}
-
-	if searchQuery.Limit > 0 {
-		values = append(values, searchQuery.Limit)
-		query += " LIMIT ?"
+	row := db.client.QueryRow(query, values...)
+	sequence := new(Sequence)
+	err := rowScanner(row.Scan, sequence)
+	if err != nil {
+		logging.Log("SQL-WsxTg").WithError(err).Info("query failed")
+		return 0, errors.ThrowInternal(err, "SQL-Yczyx", "unable to filter latest sequence")
 	}
-
-	query = numberPlaceholder(query, "?", "$")
-
-	return query, values
-}
-
-func numberPlaceholder(query, old, new string) string {
-	for i, hasChanged := 1, true; hasChanged; i++ {
-		newQuery := strings.Replace(query, old, new+strconv.Itoa(i), 1)
-		hasChanged = query != newQuery
-		query = newQuery
-	}
-	return query
-}
-
-func prepareWhere(searchQuery *es_models.SearchQuery) (clause string, values []interface{}) {
-	values = make([]interface{}, len(searchQuery.Filters))
-	clauses := make([]string, len(searchQuery.Filters))
-
-	if len(values) == 0 {
-		return clause, values
-	}
-
-	for i, filter := range searchQuery.Filters {
-		value := filter.GetValue()
-		switch value.(type) {
-		case []bool, []float64, []int64, []string, []models.AggregateType, []models.EventType, *[]bool, *[]float64, *[]int64, *[]string, *[]models.AggregateType, *[]models.EventType:
-			value = pq.Array(value)
-		}
-
-		clauses[i] = getCondition(filter)
-		values[i] = value
-	}
-	return " WHERE " + strings.Join(clauses, " AND "), values
-}
-
-func getCondition(filter *es_models.Filter) string {
-	field := getField(filter.GetField())
-	operation := getOperation(filter.GetOperation())
-	format := prepareConditionFormat(filter.GetOperation())
-
-	return fmt.Sprintf(format, field, operation)
-}
-
-func prepareConditionFormat(operation es_models.Operation) string {
-	if operation == es_models.Operation_In {
-		return "%s %s ANY(?)"
-	}
-	return "%s %s ?"
-}
-
-func getField(field es_models.Field) string {
-	switch field {
-	case es_models.Field_AggregateID:
-		return "aggregate_id"
-	case es_models.Field_AggregateType:
-		return "aggregate_type"
-	case es_models.Field_LatestSequence:
-		return "event_sequence"
-	case es_models.Field_ResourceOwner:
-		return "resource_owner"
-	case es_models.Field_EditorService:
-		return "editor_service"
-	case es_models.Field_EditorUser:
-		return "editor_user"
-	case es_models.Field_EventType:
-		return "event_type"
-	}
-	return ""
-}
-
-func getOperation(operation es_models.Operation) string {
-	switch operation {
-	case es_models.Operation_Equals, es_models.Operation_In:
-		return "="
-	case es_models.Operation_Greater:
-		return ">"
-	case es_models.Operation_Less:
-		return "<"
-	}
-	return ""
+	return uint64(*sequence), nil
 }

internal/eventstore/internal/repository/sql/filter_test.go

@@ -3,6 +3,7 @@ package sql
 import (
 	"context"
 	"database/sql"
+	"math"
 	"testing"
 
 	"github.com/caos/zitadel/internal/errors"
@@ -15,39 +16,46 @@ func TestSQL_Filter(t *testing.T) {
 	}
 	type args struct {
 		events      *mockEvents
-		searchQuery *es_models.SearchQuery
+		searchQuery *es_models.SearchQueryFactory
 	}
-	tests := []struct {
-		name      string
-		fields    fields
-		args      args
-		eventsLen int
+	type res struct {
 		wantErr   bool
 		isErrFunc func(error) bool
+		eventsLen int
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		res    res
 	}{
 		{
 			name: "only limit filter",
 			fields: fields{
-				client: mockDB(t).expectFilterEventsLimit(34, 3),
+				client: mockDB(t).expectFilterEventsLimit("user", 34, 3),
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: es_models.NewSearchQuery().SetLimit(34),
+				searchQuery: es_models.NewSearchQueryFactory("user").Limit(34),
+			},
+			res: res{
+				eventsLen: 3,
+				wantErr:   false,
 			},
-			eventsLen: 3,
-			wantErr:   false,
 		},
 		{
 			name: "only desc filter",
 			fields: fields{
-				client: mockDB(t).expectFilterEventsDesc(34),
+				client: mockDB(t).expectFilterEventsDesc("user", 34),
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: es_models.NewSearchQuery().OrderDesc(),
+				searchQuery: es_models.NewSearchQueryFactory("user").OrderDesc(),
+			},
+			res: res{
+				eventsLen: 34,
+				wantErr:   false,
 			},
-			eventsLen: 34,
-			wantErr:   false,
 		},
 		{
 			name: "no events found",
@@ -56,10 +64,12 @@ func TestSQL_Filter(t *testing.T) {
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: &es_models.SearchQuery{},
+				searchQuery: es_models.NewSearchQueryFactory("nonAggregate"),
+			},
+			res: res{
+				wantErr:   true,
+				isErrFunc: errors.IsInternal,
 			},
-			wantErr:   true,
-			isErrFunc: errors.IsInternal,
 		},
 		{
 			name: "filter fails because sql internal error",
@@ -68,34 +78,40 @@ func TestSQL_Filter(t *testing.T) {
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: &es_models.SearchQuery{},
+				searchQuery: es_models.NewSearchQueryFactory("user"),
+			},
+			res: res{
+				wantErr:   true,
+				isErrFunc: errors.IsInternal,
 			},
-			wantErr:   true,
-			isErrFunc: errors.IsInternal,
 		},
 		{
 			name: "filter by aggregate id",
 			fields: fields{
-				client: mockDB(t).expectFilterEventsAggregateIDLimit("hop", 5),
+				client: mockDB(t).expectFilterEventsAggregateIDLimit("user", "hop", 5),
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: es_models.NewSearchQuery().SetLimit(5).AggregateIDFilter("hop"),
+				searchQuery: es_models.NewSearchQueryFactory("user").Limit(5).AggregateIDs("hop"),
+			},
+			res: res{
+				wantErr:   false,
+				isErrFunc: nil,
 			},
-			wantErr:   false,
-			isErrFunc: nil,
 		},
 		{
 			name: "filter by aggregate id and aggregate type",
 			fields: fields{
-				client: mockDB(t).expectFilterEventsAggregateIDTypeLimit("hop", "user", 5),
+				client: mockDB(t).expectFilterEventsAggregateIDTypeLimit("user", "hop", 5),
 			},
 			args: args{
 				events:      &mockEvents{t: t},
-				searchQuery: es_models.NewSearchQuery().SetLimit(5).AggregateIDFilter("hop").AggregateTypeFilter("user"),
+				searchQuery: es_models.NewSearchQueryFactory("user").Limit(5).AggregateIDs("hop"),
+			},
+			res: res{
+				wantErr:   false,
+				isErrFunc: nil,
 			},
-			wantErr:   false,
-			isErrFunc: nil,
 		},
 	}
 	for _, tt := range tests {
@@ -104,13 +120,13 @@ func TestSQL_Filter(t *testing.T) {
 				client: tt.fields.client.sqlClient,
 			}
 			events, err := sql.Filter(context.Background(), tt.args.searchQuery)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("SQL.Filter() error = %v, wantErr %v", err, tt.wantErr)
+			if (err != nil) != tt.res.wantErr {
+				t.Errorf("SQL.Filter() error = %v, wantErr %v", err, tt.res.wantErr)
 			}
-			if tt.eventsLen != 0 && len(events) != tt.eventsLen {
-				t.Errorf("events has wrong length got: %d want %d", len(events), tt.eventsLen)
+			if tt.res.eventsLen != 0 && len(events) != tt.res.eventsLen {
+				t.Errorf("events has wrong length got: %d want %d", len(events), tt.res.eventsLen)
 			}
-			if tt.wantErr && !tt.isErrFunc(err) {
+			if tt.res.wantErr && !tt.res.isErrFunc(err) {
 				t.Errorf("got wrong error %v", err)
 			}
 			if err := tt.fields.client.mock.ExpectationsWereMet(); err != nil {
@@ -121,35 +137,97 @@ func TestSQL_Filter(t *testing.T) {
 	}
 }
 
-func Test_getCondition(t *testing.T) {
+func TestSQL_LatestSequence(t *testing.T) {
+	type fields struct {
+		client *dbMock
+	}
 	type args struct {
-		filter *es_models.Filter
+		searchQuery *es_models.SearchQueryFactory
+	}
+	type res struct {
+		wantErr   bool
+		isErrFunc func(error) bool
+		sequence  uint64
 	}
 	tests := []struct {
-		name string
-		args args
-		want string
+		name   string
+		fields fields
+		args   args
+		res    res
 	}{
 		{
-			name: "single value",
+			name: "invalid query factory",
 			args: args{
-				filter: es_models.NewFilter(es_models.Field_LatestSequence, 34, es_models.Operation_Greater),
+				searchQuery: nil,
+			},
+			fields: fields{
+				client: mockDB(t),
+			},
+			res: res{
+				wantErr:   true,
+				isErrFunc: errors.IsErrorInvalidArgument,
 			},
-			want: "event_sequence > ?",
 		},
 		{
-			name: "list value",
+			name: "no events for aggregate",
 			args: args{
-				filter: es_models.NewFilter(es_models.Field_AggregateType, []string{"a", "b"}, es_models.Operation_In),
+				searchQuery: es_models.NewSearchQueryFactory("idiot").Columns(es_models.Columns_Max_Sequence),
+			},
+			fields: fields{
+				client: mockDB(t).expectLatestSequenceFilterError("idiot", sql.ErrNoRows),
+			},
+			res: res{
+				wantErr:  false,
+				sequence: 0,
+			},
+		},
+		{
+			name: "sql query error",
+			args: args{
+				searchQuery: es_models.NewSearchQueryFactory("idiot").Columns(es_models.Columns_Max_Sequence),
+			},
+			fields: fields{
+				client: mockDB(t).expectLatestSequenceFilterError("idiot", sql.ErrConnDone),
+			},
+			res: res{
+				wantErr:   true,
+				isErrFunc: errors.IsInternal,
+				sequence:  0,
+			},
+		},
+		{
+			name: "events for aggregate found",
+			args: args{
+				searchQuery: es_models.NewSearchQueryFactory("user").Columns(es_models.Columns_Max_Sequence),
+			},
+			fields: fields{
+				client: mockDB(t).expectLatestSequenceFilter("user", math.MaxUint64),
+			},
+			res: res{
+				wantErr:  false,
+				sequence: math.MaxUint64,
 			},
-			want: "aggregate_type = ANY(?)",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := getCondition(tt.args.filter); got != tt.want {
-				t.Errorf("getCondition() = %v, want %v", got, tt.want)
+			sql := &SQL{
+				client: tt.fields.client.sqlClient,
 			}
+			sequence, err := sql.LatestSequence(context.Background(), tt.args.searchQuery)
+			if (err != nil) != tt.res.wantErr {
+				t.Errorf("SQL.Filter() error = %v, wantErr %v", err, tt.res.wantErr)
+			}
+			if tt.res.sequence != sequence {
+				t.Errorf("events has wrong length got: %d want %d", sequence, tt.res.sequence)
+			}
+			if tt.res.wantErr && !tt.res.isErrFunc(err) {
+				t.Errorf("got wrong error %v", err)
+			}
+			if err := tt.fields.client.mock.ExpectationsWereMet(); err != nil {
+				t.Errorf("there were unfulfilled expectations: %s", err)
+			}
+			tt.fields.client.close()
 		})
 	}
 }

internal/eventstore/internal/repository/sql/push.go

@@ -17,7 +17,7 @@ const (
 		"SELECT $1, $2, $3, $4, COALESCE($5, now()), $6, $7, $8, $9, $10 " +
 		"WHERE EXISTS (" +
 		"SELECT 1 FROM eventstore.events WHERE aggregate_type = $11 AND aggregate_id = $12 HAVING MAX(event_sequence) = $13 OR ($14::BIGINT IS NULL AND COUNT(*) = 0)) " +
-		"RETURNING id, event_sequence, creation_date"
+		"RETURNING event_sequence, creation_date"
 )
 
 func (db *SQL) PushAggregates(ctx context.Context, aggregates ...*models.Aggregate) (err error) {
@@ -55,7 +55,7 @@ func precondtion(tx *sql.Tx, aggregate *models.Aggregate) error {
 	if aggregate.Precondition == nil {
 		return nil
 	}
-	events, err := filter(tx, aggregate.Precondition.Query)
+	events, err := filter(tx, models.FactoryFromSearchQuery(aggregate.Precondition.Query))
 	if err != nil {
 		return caos_errs.ThrowPreconditionFailed(err, "SQL-oBPxB", "filter failed")
 	}
@@ -69,12 +69,11 @@ func precondtion(tx *sql.Tx, aggregate *models.Aggregate) error {
 func insertEvents(stmt *sql.Stmt, previousSequence Sequence, events []*models.Event) error {
 	for _, event := range events {
 		err := stmt.QueryRow(event.Type, event.AggregateType, event.AggregateID, event.AggregateVersion, event.CreationDate, Data(event.Data), event.EditorUser, event.EditorService, event.ResourceOwner, previousSequence,
-			event.AggregateType, event.AggregateID, previousSequence, previousSequence).Scan(&event.ID, &previousSequence, &event.CreationDate)
+			event.AggregateType, event.AggregateID, previousSequence, previousSequence).Scan(&previousSequence, &event.CreationDate)
 
 		if err != nil {
 			logging.LogWithFields("SQL-IP3js",
 				"aggregate", event.AggregateType,
-				"id", event.AggregateID,
 				"previousSequence", previousSequence,
 				"aggregateId", event.AggregateID,
 				"aggregateType", event.AggregateType,

internal/eventstore/internal/repository/sql/push_test.go

@@ -3,11 +3,12 @@ package sql
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"reflect"
 	"runtime"
 	"testing"
 
-	"github.com/caos/zitadel/internal/errors"
+	z_errors "github.com/caos/zitadel/internal/errors"
 	"github.com/caos/zitadel/internal/eventstore/models"
 )
 
@@ -36,7 +37,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
-					expectPrepareInsert().
+					expectPrepareInsert(nil).
 					expectReleaseSavepoint(nil).
 					expectCommit(nil),
 			},
@@ -44,19 +45,33 @@ func TestSQL_PushAggregates(t *testing.T) {
 			shouldCheckEvents: false,
 			isError:           noErr,
 		},
+		{
+			name: "prepare fails",
+			fields: fields{
+				client: mockDB(t).
+					expectBegin(nil).
+					expectSavepoint().
+					expectPrepareInsert(sql.ErrConnDone).
+					expectReleaseSavepoint(nil).
+					expectCommit(nil),
+			},
+			args:              args{aggregates: []*models.Aggregate{}},
+			shouldCheckEvents: false,
+			isError:           func(err error) bool { return errors.Is(err, sql.ErrConnDone) },
+		},
 		{
 			name: "no aggregates release fails",
 			fields: fields{
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
-					expectPrepareInsert().
+					expectPrepareInsert(nil).
 					expectReleaseSavepoint(sql.ErrConnDone).
 					expectCommit(nil),
 			},
 
 			args:              args{aggregates: []*models.Aggregate{}},
-			isError:           errors.IsInternal,
+			isError:           z_errors.IsInternal,
 			shouldCheckEvents: false,
 		},
 		{
@@ -65,13 +80,13 @@ func TestSQL_PushAggregates(t *testing.T) {
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
-					expectPrepareInsert().
-					expectFilterEventsError(errors.CreateCaosError(nil, "SQL-IzJOf", "err")).
+					expectPrepareInsert(nil).
+					expectFilterEventsError(z_errors.CreateCaosError(nil, "SQL-IzJOf", "err")).
 					expectRollback(nil),
 			},
 
 			args:              args{aggregates: []*models.Aggregate{aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(1), nil)}},
-			isError:           errors.IsPreconditionFailed,
+			isError:           z_errors.IsPreconditionFailed,
 			shouldCheckEvents: false,
 		},
 		{
@@ -80,7 +95,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
-					expectPrepareInsert().
+					expectPrepareInsert(nil).
 					expectInsertEvent(&models.Event{
 						AggregateID:      "aggID",
 						AggregateType:    "aggType",
@@ -90,8 +105,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 						PreviousSequence: 34,
 						Type:             "eventTyp",
 						AggregateVersion: "v0.0.1",
-					},
-						"asdfölk-234", 45).
+					}, 45).
 					expectInsertEvent(&models.Event{
 						AggregateID:      "aggID",
 						AggregateType:    "aggType",
@@ -101,7 +115,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 						PreviousSequence: 45,
 						Type:             "eventTyp",
 						AggregateVersion: "v0.0.1",
-					}, "asdfölk-233", 46).
+					}, 46).
 					expectReleaseSavepoint(nil).
 					expectCommit(nil),
 			},
@@ -141,7 +155,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
-					expectPrepareInsert().
+					expectPrepareInsert(nil).
 					expectInsertEvent(&models.Event{
 						AggregateID:      "aggID",
 						AggregateType:    "aggType",
@@ -151,7 +165,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 						PreviousSequence: 34,
 						Type:             "eventTyp",
 						AggregateVersion: "v0.0.1",
-					}, "asdfölk-233", 47).
+					}, 47).
 					expectInsertEvent(&models.Event{
 						AggregateID:      "aggID2",
 						AggregateType:    "aggType2",
@@ -161,7 +175,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 						PreviousSequence: 40,
 						Type:             "eventTyp",
 						AggregateVersion: "v0.0.1",
-					}, "asdfölk-233", 48).
+					}, 48).
 					expectReleaseSavepoint(nil).
 					expectCommit(nil),
 			},
@@ -206,6 +220,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 				client: mockDB(t).
 					expectBegin(nil).
 					expectSavepoint().
+					expectPrepareInsert(nil).
 					expectInsertEventError(&models.Event{
 						AggregateID:      "aggID",
 						AggregateType:    "aggType",
@@ -248,7 +263,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 					},
 				},
 			},
-			isError:           errors.IsInternal,
+			isError:           z_errors.IsInternal,
 			shouldCheckEvents: false,
 		},
 		{
@@ -256,7 +271,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 			fields: fields{
 				client: mockDB(t).
 					expectBegin(nil).
-					expectPrepareInsert().
+					expectPrepareInsert(nil).
 					expectSavepoint().
 					expectInsertEvent(&models.Event{
 						AggregateID:      "aggID",
@@ -268,7 +283,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 						Type:             "eventTyp",
 						Data:             []byte("{}"),
 						AggregateVersion: "v0.0.1",
-					}, "asdfölk-233", 47).
+					}, 47).
 					expectReleaseSavepoint(sql.ErrConnDone).
 					expectCommit(nil).
 					expectRollback(nil),
@@ -291,7 +306,7 @@ func TestSQL_PushAggregates(t *testing.T) {
 					},
 				},
 			},
-			isError:           errors.IsInternal,
+			isError:           z_errors.IsInternal,
 			shouldCheckEvents: false,
 		},
 	}
@@ -359,42 +374,42 @@ func Test_precondtion(t *testing.T) {
 			name: "precondition fails",
 			fields: fields{
 				client: mockDB(t).
-					expectBegin(nil).expectFilterEventsLimit(5, 0),
+					expectBegin(nil).expectFilterEventsLimit("test", 5, 0),
 			},
 			args: args{
-				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5), validationFunc(errors.ThrowPreconditionFailed(nil, "SQL-LBIKm", "err"))),
+				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5).AggregateTypeFilter("test"), validationFunc(z_errors.ThrowPreconditionFailed(nil, "SQL-LBIKm", "err"))),
 			},
-			isErr: errors.IsPreconditionFailed,
+			isErr: z_errors.IsPreconditionFailed,
 		},
 		{
 			name: "precondition with filter error",
 			fields: fields{
 				client: mockDB(t).
-					expectBegin(nil).expectFilterEventsError(errors.ThrowInternal(nil, "SQL-ac9EW", "err")),
+					expectBegin(nil).expectFilterEventsError(z_errors.ThrowInternal(nil, "SQL-ac9EW", "err")),
 			},
 			args: args{
-				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5), validationFunc(errors.CreateCaosError(nil, "SQL-LBIKm", "err"))),
+				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5).AggregateTypeFilter("test"), validationFunc(z_errors.CreateCaosError(nil, "SQL-LBIKm", "err"))),
 			},
-			isErr: errors.IsPreconditionFailed,
+			isErr: z_errors.IsPreconditionFailed,
 		},
 		{
 			name: "precondition no events",
 			fields: fields{
 				client: mockDB(t).
-					expectBegin(nil).expectFilterEventsLimit(5, 0),
+					expectBegin(nil).expectFilterEventsLimit("test", 5, 0),
 			},
 			args: args{
-				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5), validationFunc(nil)),
+				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5).AggregateTypeFilter("test"), validationFunc(nil)),
 			},
 		},
 		{
 			name: "precondition with events",
 			fields: fields{
 				client: mockDB(t).
-					expectBegin(nil).expectFilterEventsLimit(5, 3),
+					expectBegin(nil).expectFilterEventsLimit("test", 5, 3),
 			},
 			args: args{
-				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5), validationFunc(nil)),
+				aggregate: aggregateWithPrecondition(&models.Aggregate{}, models.NewSearchQuery().SetLimit(5).AggregateTypeFilter("test"), validationFunc(nil)),
 			},
 		},
 	}

internal/eventstore/internal/repository/sql/query.go

@@ -0,0 +1,199 @@
+package sql
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/caos/logging"
+	z_errors "github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore/models"
+	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/lib/pq"
+)
+
+const (
+	selectStmt = "SELECT" +
+		" creation_date" +
+		", event_type" +
+		", event_sequence" +
+		", previous_sequence" +
+		", event_data" +
+		", editor_service" +
+		", editor_user" +
+		", resource_owner" +
+		", aggregate_type" +
+		", aggregate_id" +
+		", aggregate_version" +
+		" FROM eventstore.events"
+)
+
+func buildQuery(queryFactory *models.SearchQueryFactory) (query string, limit uint64, values []interface{}, rowScanner func(s scan, dest interface{}) error) {
+	searchQuery, err := queryFactory.Build()
+	if err != nil {
+		logging.Log("SQL-cshKu").WithError(err).Warn("search query factory invalid")
+		return "", 0, nil, nil
+	}
+	query, rowScanner = prepareColumns(searchQuery.Columns)
+	where, values := prepareCondition(searchQuery.Filters)
+	if where == "" || query == "" {
+		return "", 0, nil, nil
+	}
+	query += where
+
+	if searchQuery.Columns != models.Columns_Max_Sequence {
+		query += " ORDER BY event_sequence"
+		if searchQuery.Desc {
+			query += " DESC"
+		}
+	}
+
+	if searchQuery.Limit > 0 {
+		values = append(values, searchQuery.Limit)
+		query += " LIMIT ?"
+	}
+
+	query = numberPlaceholder(query, "?", "$")
+
+	return query, searchQuery.Limit, values, rowScanner
+}
+
+func prepareCondition(filters []*models.Filter) (clause string, values []interface{}) {
+	values = make([]interface{}, len(filters))
+	clauses := make([]string, len(filters))
+
+	if len(filters) == 0 {
+		return clause, values
+	}
+	for i, filter := range filters {
+		value := filter.GetValue()
+		switch value.(type) {
+		case []bool, []float64, []int64, []string, []models.AggregateType, []models.EventType, *[]bool, *[]float64, *[]int64, *[]string, *[]models.AggregateType, *[]models.EventType:
+			value = pq.Array(value)
+		}
+
+		clauses[i] = getCondition(filter)
+		if clauses[i] == "" {
+			return "", nil
+		}
+		values[i] = value
+	}
+	return " WHERE " + strings.Join(clauses, " AND "), values
+}
+
+type scan func(dest ...interface{}) error
+
+func prepareColumns(columns models.Columns) (string, func(s scan, dest interface{}) error) {
+	switch columns {
+	case models.Columns_Max_Sequence:
+		return "SELECT MAX(event_sequence) FROM eventstore.events", func(row scan, dest interface{}) (err error) {
+			sequence, ok := dest.(*Sequence)
+			if !ok {
+				return z_errors.ThrowInvalidArgument(nil, "SQL-NBjA9", "type must be sequence")
+			}
+			err = row(sequence)
+			if err == nil || errors.Is(err, sql.ErrNoRows) {
+				return nil
+			}
+			return z_errors.ThrowInternal(err, "SQL-bN5xg", "something went wrong")
+		}
+	case models.Columns_Event:
+		return selectStmt, func(row scan, dest interface{}) (err error) {
+			event, ok := dest.(*models.Event)
+			if !ok {
+				return z_errors.ThrowInvalidArgument(nil, "SQL-4GP6F", "type must be event")
+			}
+			var previousSequence Sequence
+			data := make(Data, 0)
+
+			err = row(
+				&event.CreationDate,
+				&event.Type,
+				&event.Sequence,
+				&previousSequence,
+				&data,
+				&event.EditorService,
+				&event.EditorUser,
+				&event.ResourceOwner,
+				&event.AggregateType,
+				&event.AggregateID,
+				&event.AggregateVersion,
+			)
+
+			if err != nil {
+				logging.Log("SQL-kn1Sw").WithError(err).Warn("unable to scan row")
+				return z_errors.ThrowInternal(err, "SQL-J0hFS", "unable to scan row")
+			}
+
+			event.PreviousSequence = uint64(previousSequence)
+
+			event.Data = make([]byte, len(data))
+			copy(event.Data, data)
+
+			return nil
+		}
+	default:
+		return "", nil
+	}
+}
+
+func numberPlaceholder(query, old, new string) string {
+	for i, hasChanged := 1, true; hasChanged; i++ {
+		newQuery := strings.Replace(query, old, new+strconv.Itoa(i), 1)
+		hasChanged = query != newQuery
+		query = newQuery
+	}
+	return query
+}
+
+func getCondition(filter *es_models.Filter) (condition string) {
+	field := getField(filter.GetField())
+	operation := getOperation(filter.GetOperation())
+	if field == "" || operation == "" {
+		return ""
+	}
+	format := getConditionFormat(filter.GetOperation())
+
+	return fmt.Sprintf(format, field, operation)
+}
+
+func getConditionFormat(operation es_models.Operation) string {
+	if operation == es_models.Operation_In {
+		return "%s %s ANY(?)"
+	}
+	return "%s %s ?"
+}
+
+func getField(field es_models.Field) string {
+	switch field {
+	case es_models.Field_AggregateID:
+		return "aggregate_id"
+	case es_models.Field_AggregateType:
+		return "aggregate_type"
+	case es_models.Field_LatestSequence:
+		return "event_sequence"
+	case es_models.Field_ResourceOwner:
+		return "resource_owner"
+	case es_models.Field_EditorService:
+		return "editor_service"
+	case es_models.Field_EditorUser:
+		return "editor_user"
+	case es_models.Field_EventType:
+		return "event_type"
+	}
+	return ""
+}
+
+func getOperation(operation es_models.Operation) string {
+	switch operation {
+	case es_models.Operation_Equals, es_models.Operation_In:
+		return "="
+	case es_models.Operation_Greater:
+		return ">"
+	case es_models.Operation_Less:
+		return "<"
+	}
+	return ""
+}

internal/eventstore/internal/repository/sql/query_test.go

@@ -0,0 +1,486 @@
+package sql
+
+import (
+	"database/sql"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/caos/zitadel/internal/errors"
+	"github.com/caos/zitadel/internal/eventstore/models"
+	es_models "github.com/caos/zitadel/internal/eventstore/models"
+	"github.com/lib/pq"
+)
+
+func Test_numberPlaceholder(t *testing.T) {
+	type args struct {
+		query string
+		old   string
+		new   string
+	}
+	type res struct {
+		query string
+	}
+	tests := []struct {
+		name string
+		args args
+		res  res
+	}{
+		{
+			name: "no replaces",
+			args: args{
+				new:   "$",
+				old:   "?",
+				query: "SELECT * FROM eventstore.events",
+			},
+			res: res{
+				query: "SELECT * FROM eventstore.events",
+			},
+		},
+		{
+			name: "two replaces",
+			args: args{
+				new:   "$",
+				old:   "?",
+				query: "SELECT * FROM eventstore.events WHERE aggregate_type = ? AND LIMIT = ?",
+			},
+			res: res{
+				query: "SELECT * FROM eventstore.events WHERE aggregate_type = $1 AND LIMIT = $2",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := numberPlaceholder(tt.args.query, tt.args.old, tt.args.new); got != tt.res.query {
+				t.Errorf("numberPlaceholder() = %v, want %v", got, tt.res.query)
+			}
+		})
+	}
+}
+
+func Test_getOperation(t *testing.T) {
+	t.Run("all ops", func(t *testing.T) {
+		for op, expected := range map[es_models.Operation]string{
+			es_models.Operation_Equals:  "=",
+			es_models.Operation_In:      "=",
+			es_models.Operation_Greater: ">",
+			es_models.Operation_Less:    "<",
+			es_models.Operation(-1):     "",
+		} {
+			if got := getOperation(op); got != expected {
+				t.Errorf("getOperation() = %v, want %v", got, expected)
+			}
+		}
+	})
+}
+
+func Test_getField(t *testing.T) {
+	t.Run("all fields", func(t *testing.T) {
+		for field, expected := range map[es_models.Field]string{
+			es_models.Field_AggregateType:  "aggregate_type",
+			es_models.Field_AggregateID:    "aggregate_id",
+			es_models.Field_LatestSequence: "event_sequence",
+			es_models.Field_ResourceOwner:  "resource_owner",
+			es_models.Field_EditorService:  "editor_service",
+			es_models.Field_EditorUser:     "editor_user",
+			es_models.Field_EventType:      "event_type",
+			es_models.Field(-1):            "",
+		} {
+			if got := getField(field); got != expected {
+				t.Errorf("getField() = %v, want %v", got, expected)
+			}
+		}
+	})
+}
+
+func Test_getConditionFormat(t *testing.T) {
+	type args struct {
+		operation es_models.Operation
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "no in operation",
+			args: args{
+				operation: es_models.Operation_Equals,
+			},
+			want: "%s %s ?",
+		},
+		{
+			name: "in operation",
+			args: args{
+				operation: es_models.Operation_In,
+			},
+			want: "%s %s ANY(?)",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getConditionFormat(tt.args.operation); got != tt.want {
+				t.Errorf("prepareConditionFormat() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_getCondition(t *testing.T) {
+	type args struct {
+		filter *es_models.Filter
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "equals",
+			args: args{filter: es_models.NewFilter(es_models.Field_AggregateID, "", es_models.Operation_Equals)},
+			want: "aggregate_id = ?",
+		},
+		{
+			name: "greater",
+			args: args{filter: es_models.NewFilter(es_models.Field_LatestSequence, 0, es_models.Operation_Greater)},
+			want: "event_sequence > ?",
+		},
+		{
+			name: "less",
+			args: args{filter: es_models.NewFilter(es_models.Field_LatestSequence, 5000, es_models.Operation_Less)},
+			want: "event_sequence < ?",
+		},
+		{
+			name: "in list",
+			args: args{filter: es_models.NewFilter(es_models.Field_AggregateType, []es_models.AggregateType{"movies", "actors"}, es_models.Operation_In)},
+			want: "aggregate_type = ANY(?)",
+		},
+		{
+			name: "invalid operation",
+			args: args{filter: es_models.NewFilter(es_models.Field_AggregateType, []es_models.AggregateType{"movies", "actors"}, es_models.Operation(-1))},
+			want: "",
+		},
+		{
+			name: "invalid field",
+			args: args{filter: es_models.NewFilter(es_models.Field(-1), []es_models.AggregateType{"movies", "actors"}, es_models.Operation_Equals)},
+			want: "",
+		},
+		{
+			name: "invalid field and operation",
+			args: args{filter: es_models.NewFilter(es_models.Field(-1), []es_models.AggregateType{"movies", "actors"}, es_models.Operation(-1))},
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getCondition(tt.args.filter); got != tt.want {
+				t.Errorf("getCondition() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_prepareColumns(t *testing.T) {
+	type args struct {
+		columns models.Columns
+		dest    interface{}
+		dbErr   error
+	}
+	type res struct {
+		query    string
+		dbRow    []interface{}
+		expected interface{}
+		dbErr    func(error) bool
+	}
+	tests := []struct {
+		name string
+		args args
+		res  res
+	}{
+		{
+			name: "invalid columns",
+			args: args{columns: es_models.Columns(-1)},
+			res: res{
+				query: "",
+				dbErr: func(err error) bool { return err == nil },
+			},
+		},
+		{
+			name: "max column",
+			args: args{
+				columns: es_models.Columns_Max_Sequence,
+				dest:    new(Sequence),
+			},
+			res: res{
+				query:    "SELECT MAX(event_sequence) FROM eventstore.events",
+				dbRow:    []interface{}{Sequence(5)},
+				expected: Sequence(5),
+			},
+		},
+		{
+			name: "max sequence wrong dest type",
+			args: args{
+				columns: es_models.Columns_Max_Sequence,
+				dest:    new(uint64),
+			},
+			res: res{
+				query: "SELECT MAX(event_sequence) FROM eventstore.events",
+				dbErr: errors.IsErrorInvalidArgument,
+			},
+		},
+		{
+			name: "event",
+			args: args{
+				columns: es_models.Columns_Event,
+				dest:    new(models.Event),
+			},
+			res: res{
+				query:    "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events",
+				dbRow:    []interface{}{time.Time{}, models.EventType(""), uint64(5), Sequence(0), Data(nil), "", "", "", models.AggregateType("user"), "hodor", models.Version("")},
+				expected: models.Event{AggregateID: "hodor", AggregateType: "user", Sequence: 5, Data: make(Data, 0)},
+			},
+		},
+		{
+			name: "event wrong dest type",
+			args: args{
+				columns: es_models.Columns_Event,
+				dest:    new(uint64),
+			},
+			res: res{
+				query: "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events",
+				dbErr: errors.IsErrorInvalidArgument,
+			},
+		},
+		{
+			name: "event query error",
+			args: args{
+				columns: es_models.Columns_Event,
+				dest:    new(models.Event),
+				dbErr:   sql.ErrConnDone,
+			},
+			res: res{
+				query: "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events",
+				dbErr: errors.IsInternal,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			query, rowScanner := prepareColumns(tt.args.columns)
+			if query != tt.res.query {
+				t.Errorf("prepareColumns() got = %v, want %v", query, tt.res.query)
+			}
+			if tt.res.query == "" && rowScanner != nil {
+				t.Errorf("row scanner should be nil")
+			}
+			if rowScanner == nil {
+				return
+			}
+			err := rowScanner(prepareTestScan(tt.args.dbErr, tt.res.dbRow), tt.args.dest)
+			if tt.res.dbErr != nil {
+				if !tt.res.dbErr(err) {
+					t.Errorf("wrong error type in rowScanner got: %v", err)
+				}
+			} else {
+				if !reflect.DeepEqual(reflect.Indirect(reflect.ValueOf(tt.args.dest)).Interface(), tt.res.expected) {
+					t.Errorf("unexpected result from rowScanner want: %v got: %v", tt.res.dbRow, tt.args.dest)
+				}
+			}
+		})
+	}
+}
+
+func prepareTestScan(err error, res []interface{}) scan {
+	return func(dests ...interface{}) error {
+		if err != nil {
+			return err
+		}
+		if len(dests) != len(res) {
+			return errors.ThrowInvalidArgumentf(nil, "SQL-NML1q", "expected len %d got %d", len(res), len(dests))
+		}
+		for i, r := range res {
+			reflect.ValueOf(dests[i]).Elem().Set(reflect.ValueOf(r))
+		}
+
+		return nil
+	}
+}
+
+func Test_prepareCondition(t *testing.T) {
+	type args struct {
+		filters []*models.Filter
+	}
+	type res struct {
+		clause string
+		values []interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		res  res
+	}{
+		{
+			name: "nil filters",
+			args: args{
+				filters: nil,
+			},
+			res: res{
+				clause: "",
+				values: nil,
+			},
+		},
+		{
+			name: "empty filters",
+			args: args{
+				filters: []*es_models.Filter{},
+			},
+			res: res{
+				clause: "",
+				values: nil,
+			},
+		},
+		{
+			name: "invalid condition",
+			args: args{
+				filters: []*es_models.Filter{
+					es_models.NewFilter(es_models.Field_AggregateID, "wrong", es_models.Operation(-1)),
+				},
+			},
+			res: res{
+				clause: "",
+				values: nil,
+			},
+		},
+		{
+			name: "array as condition value",
+			args: args{
+				filters: []*es_models.Filter{
+					es_models.NewFilter(es_models.Field_AggregateType, []es_models.AggregateType{"user", "org"}, es_models.Operation_In),
+				},
+			},
+			res: res{
+				clause: " WHERE aggregate_type = ANY(?)",
+				values: []interface{}{pq.Array([]es_models.AggregateType{"user", "org"})},
+			},
+		},
+		{
+			name: "multiple filters",
+			args: args{
+				filters: []*es_models.Filter{
+					es_models.NewFilter(es_models.Field_AggregateType, []es_models.AggregateType{"user", "org"}, es_models.Operation_In),
+					es_models.NewFilter(es_models.Field_AggregateID, "1234", es_models.Operation_Equals),
+					es_models.NewFilter(es_models.Field_EventType, []es_models.EventType{"user.created", "org.created"}, es_models.Operation_In),
+				},
+			},
+			res: res{
+				clause: " WHERE aggregate_type = ANY(?) AND aggregate_id = ? AND event_type = ANY(?)",
+				values: []interface{}{pq.Array([]es_models.AggregateType{"user", "org"}), "1234", pq.Array([]es_models.EventType{"user.created", "org.created"})},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotClause, gotValues := prepareCondition(tt.args.filters)
+			if gotClause != tt.res.clause {
+				t.Errorf("prepareCondition() gotClause = %v, want %v", gotClause, tt.res.clause)
+			}
+			if len(gotValues) != len(tt.res.values) {
+				t.Errorf("wrong length of gotten values got = %d, want %d", len(gotValues), len(tt.res.values))
+				return
+			}
+			for i, value := range gotValues {
+				if !reflect.DeepEqual(value, tt.res.values[i]) {
+					t.Errorf("prepareCondition() gotValues = %v, want %v", gotValues, tt.res.values)
+				}
+			}
+		})
+	}
+}
+
+func Test_buildQuery(t *testing.T) {
+	type args struct {
+		queryFactory *models.SearchQueryFactory
+	}
+	type res struct {
+		query      string
+		limit      uint64
+		values     []interface{}
+		rowScanner bool
+	}
+	tests := []struct {
+		name string
+		args args
+		res  res
+	}{
+		{
+			name: "invalid query factory",
+			args: args{
+				queryFactory: nil,
+			},
+			res: res{
+				query:      "",
+				limit:      0,
+				rowScanner: false,
+				values:     nil,
+			},
+		},
+		{
+			name: "with order by desc",
+			args: args{
+				queryFactory: es_models.NewSearchQueryFactory("user").OrderDesc(),
+			},
+			res: res{
+				query:      "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = $1 ORDER BY event_sequence DESC",
+				rowScanner: true,
+				values:     []interface{}{es_models.AggregateType("user")},
+			},
+		},
+		{
+			name: "with limit",
+			args: args{
+				queryFactory: es_models.NewSearchQueryFactory("user").Limit(5),
+			},
+			res: res{
+				query:      "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = $1 ORDER BY event_sequence LIMIT $2",
+				rowScanner: true,
+				values:     []interface{}{es_models.AggregateType("user"), uint64(5)},
+				limit:      5,
+			},
+		},
+		{
+			name: "with limit and order by desc",
+			args: args{
+				queryFactory: es_models.NewSearchQueryFactory("user").Limit(5).OrderDesc(),
+			},
+			res: res{
+				query:      "SELECT creation_date, event_type, event_sequence, previous_sequence, event_data, editor_service, editor_user, resource_owner, aggregate_type, aggregate_id, aggregate_version FROM eventstore.events WHERE aggregate_type = $1 ORDER BY event_sequence DESC LIMIT $2",
+				rowScanner: true,
+				values:     []interface{}{es_models.AggregateType("user"), uint64(5)},
+				limit:      5,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotQuery, gotLimit, gotValues, gotRowScanner := buildQuery(tt.args.queryFactory)
+			if gotQuery != tt.res.query {
+				t.Errorf("buildQuery() gotQuery = %v, want %v", gotQuery, tt.res.query)
+			}
+			if gotLimit != tt.res.limit {
+				t.Errorf("buildQuery() gotLimit = %v, want %v", gotLimit, tt.res.limit)
+			}
+			if len(gotValues) != len(tt.res.values) {
+				t.Errorf("wrong length of gotten values got = %d, want %d", len(gotValues), len(tt.res.values))
+				return
+			}
+			for i, value := range gotValues {
+				if !reflect.DeepEqual(value, tt.res.values[i]) {
+					t.Errorf("prepareCondition() gotValues = %v, want %v", gotValues, tt.res.values)
+				}
+			}
+			if (tt.res.rowScanner && gotRowScanner == nil) || (!tt.res.rowScanner && gotRowScanner != nil) {
+				t.Errorf("rowScanner should be nil==%v got nil==%v", tt.res.rowScanner, gotRowScanner == nil)
+			}
+		})
+	}
+}

internal/eventstore/internal/repository/sql/sequence.go

@@ -1,27 +0,0 @@
-package sql
-
-import (
-	"database/sql/driver"
-)
-
-// Sequence represents a number that may be null.
-// Sequence implements the sql.Scanner interface
-type Sequence uint64
-
-// Scan implements the Scanner interface.
-func (seq *Sequence) Scan(value interface{}) error {
-	if value == nil {
-		*seq = 0
-		return nil
-	}
-	*seq = Sequence(value.(int64))
-	return nil
-}
-
-// Value implements the driver Valuer interface.
-func (seq Sequence) Value() (driver.Value, error) {
-	if seq == 0 {
-		return nil, nil
-	}
-	return int64(seq), nil
-}

internal/eventstore/internal/repository/sql/types.go

@@ -23,3 +23,25 @@ func (data Data) Value() (driver.Value, error) {
 	}
 	return []byte(data), nil
 }
+
+// Sequence represents a number that may be null.
+// Sequence implements the sql.Scanner interface
+type Sequence uint64
+
+// Scan implements the Scanner interface.
+func (seq *Sequence) Scan(value interface{}) error {
+	if value == nil {
+		*seq = 0
+		return nil
+	}
+	*seq = Sequence(value.(int64))
+	return nil
+}
+
+// Value implements the driver Valuer interface.
+func (seq Sequence) Value() (driver.Value, error) {
+	if seq == 0 {
+		return nil, nil
+	}
+	return int64(seq), nil
+}