feat: list users scim v2 endpoint (#9187)

# Which Problems Are Solved
- Adds support for the list users SCIM v2 endpoint

# How the Problems Are Solved
- Adds support for the list users SCIM v2 endpoints under `GET
/scim/v2/{orgID}/Users` and `POST /scim/v2/{orgID}/Users/.search`

# Additional Changes
- adds a new function `SearchUserMetadataForUsers` to the query layer to
query a metadata keyset for given user ids
- adds a new function `NewUserMetadataExistsQuery` to the query layer to
query a given metadata key value pair exists
- adds a new function `CountUsers` to the query layer to count users
without reading any rows
- handle `ErrorAlreadyExists` as scim errors `uniqueness`
- adds `NumberLessOrEqual` and `NumberGreaterOrEqual` query comparison
methods
- adds `BytesQuery` with `BytesEquals` and `BytesNotEquals` query
comparison methods

# Additional Context
Part of #8140
Supported fields for scim filters:
* `meta.created`
* `meta.lastModified`
* `id`
* `username`
* `name.familyName`
* `name.givenName`
* `emails` and `emails.value`
* `active` only eq and ne
* `externalId` only eq and ne

internal/query/prepare_test.go

@@ -110,7 +110,7 @@ func mockQueries(stmt string, cols []string, rows [][]driver.Value, args ...driv
 		result := m.NewRows(cols)
 		count := uint64(len(rows))
 		for _, row := range rows {
-			if cols[len(cols)-1] == "count" {
+			if cols[len(cols)-1] == "count" && len(row) == len(cols)-1 {
 				row = append(row, count)
 			}
 			result.AddRow(row...)

internal/query/search_query.go

@@ -109,6 +109,10 @@ func NewOrQuery(queries ...SearchQuery) (*OrQuery, error) {
 	return &OrQuery{queries: queries}, nil
 }
 
+func (q *OrQuery) Prepend(queries ...SearchQuery) {
+	q.queries = append(queries, q.queries...)
+}
+
 func (q *OrQuery) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
 	return query.Where(q.comp())
 }
@@ -147,6 +151,10 @@ func (q *AndQuery) comp() sq.Sqlizer {
 	return and
 }
 
+func (q *AndQuery) Prepend(queries ...SearchQuery) {
+	q.queries = append(queries, q.queries...)
+}
+
 type NotQuery struct {
 	query SearchQuery
 }
@@ -406,8 +414,12 @@ func (q *NumberQuery) comp() sq.Sqlizer {
 		return sq.NotEq{q.Column.identifier(): q.Number}
 	case NumberLess:
 		return sq.Lt{q.Column.identifier(): q.Number}
+	case NumberLessOrEqual:
+		return sq.LtOrEq{q.Column.identifier(): q.Number}
 	case NumberGreater:
 		return sq.Gt{q.Column.identifier(): q.Number}
+	case NumberGreaterOrEqual:
+		return sq.GtOrEq{q.Column.identifier(): q.Number}
 	case NumberListContains:
 		return &listContains{col: q.Column, args: []interface{}{q.Number}}
 	case numberCompareMax:
@@ -423,7 +435,9 @@ const (
 	NumberEquals NumberComparison = iota
 	NumberNotEquals
 	NumberLess
+	NumberLessOrEqual
 	NumberGreater
+	NumberGreaterOrEqual
 	NumberListContains
 
 	numberCompareMax
@@ -588,6 +602,57 @@ func (q *BoolQuery) comp() sq.Sqlizer {
 	return sq.Eq{q.Column.identifier(): q.Value}
 }
 
+type BytesComparison int
+
+const (
+	BytesEquals BytesComparison = iota
+	BytesNotEquals
+	bytesCompareMax
+)
+
+type BytesQuery struct {
+	Column  Column
+	Compare BytesComparison
+	Value   []byte
+}
+
+func NewBytesQuery(col Column, values []byte, comparison BytesComparison) (*BytesQuery, error) {
+	if col.isZero() {
+		return nil, ErrMissingColumn
+	}
+
+	if comparison < 0 || comparison >= bytesCompareMax {
+		return nil, ErrInvalidCompare
+	}
+
+	return &BytesQuery{
+		Column:  col,
+		Value:   values,
+		Compare: comparison,
+	}, nil
+}
+
+func (q *BytesQuery) Col() Column {
+	return q.Column
+}
+
+func (q *BytesQuery) toQuery(query sq.SelectBuilder) sq.SelectBuilder {
+	return query.Where(q.comp())
+}
+
+func (q *BytesQuery) comp() sq.Sqlizer {
+	switch q.Compare {
+	case BytesEquals:
+		return sq.Eq{q.Column.identifier(): q.Value}
+	case BytesNotEquals:
+		return sq.NotEq{q.Column.identifier(): q.Value}
+	case bytesCompareMax:
+		return nil
+	}
+
+	return nil
+}
+
 type TimestampComparison int
 
 const (

internal/query/search_query_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	sq "github.com/Masterminds/squirrel"
+	"github.com/stretchr/testify/require"
 
 	"github.com/zitadel/zitadel/internal/domain"
 )
@@ -1540,6 +1541,17 @@ func TestNumberQuery_comp(t *testing.T) {
 				query: sq.Lt{"test_table.test_col": 42},
 			},
 		},
+		{
+			name: "less or equal",
+			fields: fields{
+				Column:  testCol,
+				Number:  42,
+				Compare: NumberLessOrEqual,
+			},
+			want: want{
+				query: sq.LtOrEq{"test_table.test_col": 42},
+			},
+		},
 		{
 			name: "greater",
 			fields: fields{
@@ -1551,6 +1563,17 @@ func TestNumberQuery_comp(t *testing.T) {
 				query: sq.Gt{"test_table.test_col": 42},
 			},
 		},
+		{
+			name: "greater or equal",
+			fields: fields{
+				Column:  testCol,
+				Number:  42,
+				Compare: NumberGreaterOrEqual,
+			},
+			want: want{
+				query: sq.GtOrEq{"test_table.test_col": 42},
+			},
+		},
 		{
 			name: "list containts",
 			fields: fields{
@@ -2193,3 +2216,98 @@ func TestInTextQuery_comp(t *testing.T) {
 		})
 	}
 }
+
+func TestBytesQuery_comp(t *testing.T) {
+	type fields struct {
+		Column  Column
+		Value   []byte
+		Compare BytesComparison
+	}
+	type want struct {
+		query interface{}
+		err   bool
+		isNil bool
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   want
+	}{
+		{
+			name: "equals",
+			fields: fields{
+				Column:  testCol,
+				Value:   []byte("foo"),
+				Compare: BytesEquals,
+			},
+			want: want{
+				query: sq.Eq{"test_table.test_col": []byte("foo")},
+			},
+		},
+		{
+			name: "not equals",
+			fields: fields{
+				Column:  testCol,
+				Value:   []byte("foo"),
+				Compare: BytesNotEquals,
+			},
+			want: want{
+				query: sq.NotEq{"test_table.test_col": []byte("foo")},
+			},
+		},
+		{
+			name: "unknown comparison",
+			fields: fields{
+				Column:  testCol,
+				Value:   []byte("foo"),
+				Compare: -1,
+			},
+			want: want{
+				err:   true,
+				isNil: true,
+			},
+		},
+		{
+			name: "zero col",
+			fields: fields{
+				Column:  Column{},
+				Value:   []byte("foo"),
+				Compare: BytesEquals,
+			},
+			want: want{
+				err:   true,
+				query: sq.Eq{"": []byte("foo")},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, err := NewBytesQuery(tt.fields.Column, tt.fields.Value, tt.fields.Compare)
+			if tt.want.err {
+				require.Error(t, err)
+
+				// still test comp
+				s = &BytesQuery{
+					Column:  tt.fields.Column,
+					Value:   tt.fields.Value,
+					Compare: tt.fields.Compare,
+				}
+			} else {
+				require.NoError(t, err)
+			}
+
+			query := s.comp()
+
+			if tt.want.isNil {
+				require.Nil(t, query)
+				return
+			}
+
+			require.NotNil(t, query)
+
+			if !reflect.DeepEqual(query, tt.want.query) {
+				t.Errorf("wrong query: want: %v, (%T), got: %v, (%T)", tt.want.query, tt.want.query, query, query)
+			}
+		})
+	}
+}

internal/query/user.go

@@ -604,6 +604,27 @@ func (q *Queries) GetNotifyUser(ctx context.Context, shouldTriggered bool, queri
 	return user, err
 }
 
+func (q *Queries) CountUsers(ctx context.Context, queries *UserSearchQueries) (count uint64, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	query, scan := prepareCountUsersQuery()
+	eq := sq.Eq{UserInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID()}
+	stmt, args, err := queries.toQuery(query).Where(eq).ToSql()
+	if err != nil {
+		return 0, zerrors.ThrowInternal(err, "QUERY-w3Dx", "Errors.Query.SQLStatment")
+	}
+
+	err = q.client.QueryContext(ctx, func(rows *sql.Rows) error {
+		count, err = scan(rows)
+		return err
+	}, stmt, args...)
+	if err != nil {
+		return 0, zerrors.ThrowInternal(err, "QUERY-AG4gs", "Errors.Internal")
+	}
+	return count, err
+}
+
 func (q *Queries) SearchUsers(ctx context.Context, queries *UserSearchQueries, permissionCheck domain.PermissionCheck) (*Users, error) {
 	users, err := q.searchUsers(ctx, queries, permissionCheck != nil && authz.GetFeatures(ctx).PermissionCheckV2)
 	if err != nil {
@@ -1278,6 +1299,24 @@ func scanNotifyUser(row *sql.Row) (*NotifyUser, error) {
 	return u, nil
 }
 
+func prepareCountUsersQuery() (sq.SelectBuilder, func(*sql.Rows) (uint64, error)) {
+	return sq.Select(countColumn.identifier()).
+			From(userTable.identifier()).
+			LeftJoin(join(HumanUserIDCol, UserIDCol)).
+			LeftJoin(join(MachineUserIDCol, UserIDCol)).
+			PlaceholderFormat(sq.Dollar),
+		func(rows *sql.Rows) (count uint64, err error) {
+			// the count is implemented as a windowing function,
+			// if it is zero, no row is returned at all.
+			if !rows.Next() {
+				return
+			}
+
+			err = rows.Scan(&count)
+			return
+		}
+}
+
 func prepareUserUniqueQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Row) (bool, error)) {
 	return sq.Select(
 			UserIDCol.identifier(),

internal/query/user_metadata.go

@@ -24,6 +24,7 @@ type UserMetadataList struct {
 
 type UserMetadata struct {
 	CreationDate  time.Time `json:"creation_date,omitempty"`
+	UserID        string    `json:"-"`
 	ChangeDate    time.Time `json:"change_date,omitempty"`
 	ResourceOwner string    `json:"resource_owner,omitempty"`
 	Sequence      uint64    `json:"sequence,omitempty"`
@@ -107,6 +108,38 @@ func (q *Queries) GetUserMetadataByKey(ctx context.Context, shouldTriggerBulk bo
 	return metadata, err
 }
 
+func (q *Queries) SearchUserMetadataForUsers(ctx context.Context, shouldTriggerBulk bool, userIDs []string, queries *UserMetadataSearchQueries) (metadata *UserMetadataList, err error) {
+	ctx, span := tracing.NewSpan(ctx)
+	defer func() { span.EndWithError(err) }()
+
+	if shouldTriggerBulk {
+		_, traceSpan := tracing.NewNamedSpan(ctx, "TriggerUserMetadataProjection")
+		ctx, err = projection.UserMetadataProjection.Trigger(ctx, handler.WithAwaitRunning())
+		logging.OnError(err).Debug("trigger failed")
+		traceSpan.EndWithError(err)
+	}
+
+	query, scan := prepareUserMetadataListQuery(ctx, q.client)
+	eq := sq.Eq{
+		UserMetadataUserIDCol.identifier():     userIDs,
+		UserMetadataInstanceIDCol.identifier(): authz.GetInstance(ctx).InstanceID(),
+	}
+	stmt, args, err := queries.toQuery(query).Where(eq).ToSql()
+	if err != nil {
+		return nil, zerrors.ThrowInternal(err, "QUERY-Egbgd", "Errors.Query.SQLStatment")
+	}
+
+	err = q.client.QueryContext(ctx, func(rows *sql.Rows) error {
+		metadata, err = scan(rows)
+		return err
+	}, stmt, args...)
+	if err != nil {
+		return nil, err
+	}
+	metadata.State, err = q.latestState(ctx, userMetadataTable)
+	return metadata, err
+}
+
 func (q *Queries) SearchUserMetadata(ctx context.Context, shouldTriggerBulk bool, userID string, queries *UserMetadataSearchQueries, withOwnerRemoved bool) (metadata *UserMetadataList, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() { span.EndWithError(err) }()
@@ -164,6 +197,44 @@ func NewUserMetadataKeySearchQuery(value string, comparison TextComparison) (Sea
 	return NewTextQuery(UserMetadataKeyCol, value, comparison)
 }
 
+func NewUserMetadataExistsQuery(key string, value []byte, keyComparison TextComparison, valueComparison BytesComparison) (SearchQuery, error) {
+	// linking queries for the subselect
+	instanceQuery, err := NewColumnComparisonQuery(UserMetadataInstanceIDCol, UserInstanceIDCol, ColumnEquals)
+	if err != nil {
+		return nil, err
+	}
+
+	userIDQuery, err := NewColumnComparisonQuery(UserMetadataUserIDCol, UserIDCol, ColumnEquals)
+	if err != nil {
+		return nil, err
+	}
+
+	// text query to select data from the linked sub select
+	metadataKeyQuery, err := NewTextQuery(UserMetadataKeyCol, key, keyComparison)
+	if err != nil {
+		return nil, err
+	}
+
+	// text query to select data from the linked sub select
+	metadataValueQuery, err := NewBytesQuery(UserMetadataValueCol, value, valueComparison)
+	if err != nil {
+		return nil, err
+	}
+
+	// full definition of the sub select
+	subSelect, err := NewSubSelect(UserMetadataUserIDCol, []SearchQuery{instanceQuery, userIDQuery, metadataKeyQuery, metadataValueQuery})
+	if err != nil {
+		return nil, err
+	}
+
+	// "WHERE * IN (*)" query with subquery as list-data provider
+	return NewListQuery(
+		UserIDCol,
+		subSelect,
+		ListIn,
+	)
+}
+
 func prepareUserMetadataQuery(ctx context.Context, db prepareDatabase) (sq.SelectBuilder, func(*sql.Row) (*UserMetadata, error)) {
 	return sq.Select(
 			UserMetadataCreationDateCol.identifier(),
@@ -200,6 +271,7 @@ func prepareUserMetadataListQuery(ctx context.Context, db prepareDatabase) (sq.S
 	return sq.Select(
 			UserMetadataCreationDateCol.identifier(),
 			UserMetadataChangeDateCol.identifier(),
+			UserMetadataUserIDCol.identifier(),
 			UserMetadataResourceOwnerCol.identifier(),
 			UserMetadataSequenceCol.identifier(),
 			UserMetadataKeyCol.identifier(),
@@ -215,6 +287,7 @@ func prepareUserMetadataListQuery(ctx context.Context, db prepareDatabase) (sq.S
 				err := rows.Scan(
 					&m.CreationDate,
 					&m.ChangeDate,
+					&m.UserID,
 					&m.ResourceOwner,
 					&m.Sequence,
 					&m.Key,

internal/query/user_metadata_test.go

@@ -30,6 +30,7 @@ var (
 	}
 	userMetadataListQuery = `SELECT projections.user_metadata5.creation_date,` +
 		` projections.user_metadata5.change_date,` +
+		` projections.user_metadata5.user_id,` +
 		` projections.user_metadata5.resource_owner,` +
 		` projections.user_metadata5.sequence,` +
 		` projections.user_metadata5.key,` +
@@ -39,6 +40,7 @@ var (
 	userMetadataListCols = []string{
 		"creation_date",
 		"change_date",
+		"user_id",
 		"resource_owner",
 		"sequence",
 		"key",
@@ -148,6 +150,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 						{
 							testNow,
 							testNow,
+							"1",
 							"resource_owner",
 							uint64(20211108),
 							"key",
@@ -164,6 +167,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 					{
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
+						UserID:        "1",
 						ResourceOwner: "resource_owner",
 						Sequence:      20211108,
 						Key:           "key",
@@ -183,6 +187,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 						{
 							testNow,
 							testNow,
+							"1",
 							"resource_owner",
 							uint64(20211108),
 							"key",
@@ -191,6 +196,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 						{
 							testNow,
 							testNow,
+							"2",
 							"resource_owner",
 							uint64(20211108),
 							"key2",
@@ -207,6 +213,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 					{
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
+						UserID:        "1",
 						ResourceOwner: "resource_owner",
 						Sequence:      20211108,
 						Key:           "key",
@@ -215,6 +222,7 @@ func Test_UserMetadataPrepares(t *testing.T) {
 					{
 						CreationDate:  testNow,
 						ChangeDate:    testNow,
+						UserID:        "2",
 						ResourceOwner: "resource_owner",
 						Sequence:      20211108,
 						Key:           "key2",

internal/query/user_test.go

@@ -6,6 +6,7 @@ import (
 	"database/sql/driver"
 	"errors"
 	"fmt"
+	"reflect"
 	"regexp"
 	"testing"
 
@@ -530,6 +531,8 @@ var (
 		"access_token_type",
 		"count",
 	}
+	countUsersQuery = "SELECT COUNT(*) OVER () FROM projections.users13"
+	countUsersCols  = []string{"count"}
 )
 
 func Test_UserPrepares(t *testing.T) {
@@ -1508,10 +1511,67 @@ func Test_UserPrepares(t *testing.T) {
 			},
 			object: (*Users)(nil),
 		},
+		{
+			name:    "prepareCountUsersQuery no result",
+			prepare: prepareCountUsersQuery,
+			want: want{
+				sqlExpectations: mockQuery(
+					regexp.QuoteMeta(countUsersQuery),
+					nil,
+					nil,
+				),
+			},
+			object: uint64(0),
+		},
+		{
+			name:    "prepareCountUsersQuery one result",
+			prepare: prepareCountUsersQuery,
+			want: want{
+				sqlExpectations: mockQueries(
+					regexp.QuoteMeta(countUsersQuery),
+					countUsersCols,
+					[][]driver.Value{{uint64(1)}},
+				),
+			},
+			object: uint64(1),
+		},
+		{
+			name:    "prepareCountUsersQuery multiple results",
+			prepare: prepareCountUsersQuery,
+			want: want{
+				sqlExpectations: mockQueries(
+					regexp.QuoteMeta(countUsersQuery),
+					countUsersCols,
+					[][]driver.Value{{uint64(2)}},
+				),
+			},
+			object: uint64(2),
+		},
+		{
+			name:    "prepareCountUsersQuery sql err",
+			prepare: prepareCountUsersQuery,
+			want: want{
+				sqlExpectations: mockQueryErr(
+					regexp.QuoteMeta(countUsersQuery),
+					sql.ErrConnDone,
+				),
+				err: func(err error) (error, bool) {
+					if !errors.Is(err, sql.ErrConnDone) {
+						return fmt.Errorf("err should be sql.ErrConnDone got: %w", err), false
+					}
+					return nil, true
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err, defaultPrepareArgs...)
+			params := defaultPrepareArgs
+			if reflect.TypeOf(tt.prepare).NumIn() == 0 {
+				params = []reflect.Value{}
+			}
+
+			assertPrepare(t, tt.prepare, tt.object, tt.want.sqlExpectations, tt.want.err, params...)
 		})
 	}
 }