chore: fix some eventual consistent integration testing (#10752)

# Which Problems Are Solved

Flakiness in integration tests because of eventual consistentcy.

# How the Problems Are Solved

Split tests related to feature flags and other eventual consistent
resources.

# Additional Changes

None

# Additional Context

None

Co-authored-by: Marco A. <[email protected]>
This commit is contained in:
Stefan Benz
2025-10-14 18:24:09 +02:00
committed by GitHub
co-authored by Marco A.
parent 15bcd46709
commit a8bbac37d9
21 changed files with 2100 additions and 731 deletions
@@ -135,7 +135,7 @@ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
now := time.Now()
err := tt.args.prepare(tt.args.req)
require.NoError(t, err)
got, err := Client.AddKey(CTX, tt.args.req)
got, err := Client.AddKey(OrgCTX, tt.args.req)
if tt.wantErr {
require.Error(t, err)
return
@@ -155,7 +155,7 @@ abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789
}
func TestServer_AddKey_Permission(t *testing.T) {
OrgCTX := CTX
OrgCTX := OrgCTX
otherOrg := Instance.CreateOrganization(IamCTX, integration.OrganizationName(), integration.Email())
otherOrgUser, err := Client.CreateUser(IamCTX, &user.CreateUserRequest{
OrganizationId: otherOrg.OrganizationId,
@@ -281,7 +281,7 @@ func TestServer_RemoveKey(t *testing.T) {
now := time.Now()
err := tt.args.prepare(tt.args.req)
require.NoError(t, err)
got, err := Client.RemoveKey(CTX, tt.args.req)
got, err := Client.RemoveKey(OrgCTX, tt.args.req)
if tt.wantErr {
require.Error(t, err)
return
@@ -295,7 +295,7 @@ func TestServer_RemoveKey(t *testing.T) {
}
func TestServer_RemoveKey_Permission(t *testing.T) {
OrgCTX := CTX
OrgCTX := OrgCTX
otherOrg := Instance.CreateOrganization(IamCTX, integration.OrganizationName(), integration.Email())
otherOrgUser, err := Client.CreateUser(IamCTX, &user.CreateUserRequest{
OrganizationId: otherOrg.OrganizationId,
@@ -374,9 +374,9 @@ func TestServer_ListKeys(t *testing.T) {
args args
want *user.ListKeysResponse
}
OrgCTX := CTX
otherOrg := Instance.CreateOrganization(SystemCTX, integration.OrganizationName(), integration.Email())
otherOrgUser, err := Client.CreateUser(SystemCTX, &user.CreateUserRequest{
OrgCTX := OrgCTX
otherOrg := Instance.CreateOrganization(IamCTX, integration.OrganizationName(), integration.Email())
otherOrgUser, err := Client.CreateUser(IamCTX, &user.CreateUserRequest{
OrganizationId: otherOrg.OrganizationId,
UserType: &user.CreateUserRequest_Machine_{
Machine: &user.CreateUserRequest_Machine{
@@ -386,7 +386,7 @@ func TestServer_ListKeys(t *testing.T) {
})
require.NoError(t, err)
otherOrgUserId := otherOrgUser.GetId()
otherUserId := Instance.CreateUserTypeMachine(SystemCTX, Instance.DefaultOrg.Id).GetId()
otherUserId := Instance.CreateUserTypeMachine(IamCTX, Instance.DefaultOrg.Id).GetId()
onlySinceTestStartFilter := &user.KeysSearchFilter{Filter: &user.KeysSearchFilter_CreatedDateFilter{CreatedDateFilter: &filter.TimestampFilter{
Timestamp: timestamppb.Now(),
Method: filter.TimestampFilterMethod_TIMESTAMP_FILTER_METHOD_AFTER_OR_EQUALS,
@@ -394,12 +394,12 @@ func TestServer_ListKeys(t *testing.T) {
myOrgId := Instance.DefaultOrg.GetId()
myUserId := Instance.Users.Get(integration.UserTypeNoPermission).ID
expiresInADay := time.Now().Truncate(time.Hour).Add(time.Hour * 24)
myDataPoint := setupKeyDataPoint(t, myUserId, myOrgId, expiresInADay)
otherUserDataPoint := setupKeyDataPoint(t, otherUserId, myOrgId, expiresInADay)
otherOrgDataPointExpiringSoon := setupKeyDataPoint(t, otherOrgUserId, otherOrg.OrganizationId, time.Now().Truncate(time.Hour).Add(time.Hour))
otherOrgDataPointExpiringLate := setupKeyDataPoint(t, otherOrgUserId, otherOrg.OrganizationId, expiresInADay.Add(time.Hour*24*30))
myDataPoint := setupKeyDataPoint(IamCTX, t, Instance, myUserId, myOrgId, expiresInADay)
otherUserDataPoint := setupKeyDataPoint(IamCTX, t, Instance, otherUserId, myOrgId, expiresInADay)
otherOrgDataPointExpiringSoon := setupKeyDataPoint(IamCTX, t, Instance, otherOrgUserId, otherOrg.OrganizationId, time.Now().Truncate(time.Hour).Add(time.Hour))
otherOrgDataPointExpiringLate := setupKeyDataPoint(IamCTX, t, Instance, otherOrgUserId, otherOrg.OrganizationId, expiresInADay.Add(time.Hour*24*30))
sortingColumnExpirationDate := user.KeyFieldName_KEY_FIELD_NAME_KEY_EXPIRATION_DATE
awaitKeys(t, onlySinceTestStartFilter,
awaitKeys(IamCTX, t, Instance, onlySinceTestStartFilter,
otherOrgDataPointExpiringSoon.GetId(),
otherOrgDataPointExpiringLate.GetId(),
otherUserDataPoint.GetId(),
@@ -437,7 +437,7 @@ func TestServer_ListKeys(t *testing.T) {
myDataPoint,
},
Pagination: &filter.PaginationResponse{
TotalResult: 2,
TotalResult: 4,
AppliedLimit: 100,
},
},
@@ -453,7 +453,7 @@ func TestServer_ListKeys(t *testing.T) {
myDataPoint,
},
Pagination: &filter.PaginationResponse{
TotalResult: 1,
TotalResult: 4,
AppliedLimit: 100,
},
},
@@ -575,6 +575,245 @@ func TestServer_ListKeys(t *testing.T) {
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{},
Pagination: &filter.PaginationResponse{
TotalResult: 1,
AppliedLimit: 100,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(tt.args.ctx, 20*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := Client.ListKeys(tt.args.ctx, tt.args.req)
require.NoError(ttt, err)
if !assert.Len(ttt, got.Result, len(tt.want.Result)) {
return
}
if diff := cmp.Diff(tt.want, got, protocmp.Transform()); diff != "" {
ttt.Errorf("ListKeys() mismatch (-want +got):\n%s", diff)
}
}, retryDuration, tick, "timeout waiting for expected user result")
})
}
}
func TestServer_ListKeys_PermissionV2(t *testing.T) {
ensureFeaturePermissionV2Enabled(t, InstancePermissionV2)
iamOwnerCtx := InstancePermissionV2.WithAuthorizationToken(OrgCTX, integration.UserTypeIAMOwner)
type args struct {
ctx context.Context
req *user.ListKeysRequest
}
type testCase struct {
name string
args args
want *user.ListKeysResponse
}
otherOrg := InstancePermissionV2.CreateOrganization(iamOwnerCtx, integration.OrganizationName(), integration.Email())
otherOrgUser, err := InstancePermissionV2.Client.UserV2.CreateUser(iamOwnerCtx, &user.CreateUserRequest{
OrganizationId: otherOrg.OrganizationId,
UserType: &user.CreateUserRequest_Machine_{
Machine: &user.CreateUserRequest_Machine{
Name: integration.Username(),
},
},
})
require.NoError(t, err)
otherOrgUserId := otherOrgUser.GetId()
otherUserId := InstancePermissionV2.CreateUserTypeMachine(iamOwnerCtx, InstancePermissionV2.DefaultOrg.Id).GetId()
onlySinceTestStartFilter := &user.KeysSearchFilter{Filter: &user.KeysSearchFilter_CreatedDateFilter{CreatedDateFilter: &filter.TimestampFilter{
Timestamp: timestamppb.Now(),
Method: filter.TimestampFilterMethod_TIMESTAMP_FILTER_METHOD_AFTER_OR_EQUALS,
}}}
myOrgId := InstancePermissionV2.DefaultOrg.GetId()
myUserId := InstancePermissionV2.Users.Get(integration.UserTypeNoPermission).ID
expiresInADay := time.Now().Truncate(time.Hour).Add(time.Hour * 24)
myDataPoint := setupKeyDataPoint(iamOwnerCtx, t, InstancePermissionV2, myUserId, myOrgId, expiresInADay)
otherUserDataPoint := setupKeyDataPoint(iamOwnerCtx, t, InstancePermissionV2, otherUserId, myOrgId, expiresInADay)
otherOrgDataPointExpiringSoon := setupKeyDataPoint(iamOwnerCtx, t, InstancePermissionV2, otherOrgUserId, otherOrg.OrganizationId, time.Now().Truncate(time.Hour).Add(time.Hour))
otherOrgDataPointExpiringLate := setupKeyDataPoint(iamOwnerCtx, t, InstancePermissionV2, otherOrgUserId, otherOrg.OrganizationId, expiresInADay.Add(time.Hour*24*30))
sortingColumnExpirationDate := user.KeyFieldName_KEY_FIELD_NAME_KEY_EXPIRATION_DATE
awaitKeys(iamOwnerCtx, t, InstancePermissionV2, onlySinceTestStartFilter,
otherOrgDataPointExpiringSoon.GetId(),
otherOrgDataPointExpiringLate.GetId(),
otherUserDataPoint.GetId(),
myDataPoint.GetId(),
)
tests := []testCase{
{
name: "list all, InstancePermissionV2",
args: args{
iamOwnerCtx,
&user.ListKeysRequest{Filters: []*user.KeysSearchFilter{onlySinceTestStartFilter}},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherOrgDataPointExpiringLate,
otherOrgDataPointExpiringSoon,
otherUserDataPoint,
myDataPoint,
},
Pagination: &filter.PaginationResponse{
TotalResult: 4,
AppliedLimit: 100,
},
},
},
{
name: "list all, org",
args: args{
InstancePermissionV2.WithAuthorizationToken(iamOwnerCtx, integration.UserTypeOrgOwner),
&user.ListKeysRequest{Filters: []*user.KeysSearchFilter{onlySinceTestStartFilter}},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherUserDataPoint,
myDataPoint,
},
Pagination: &filter.PaginationResponse{
TotalResult: 2,
AppliedLimit: 100,
},
},
},
{
name: "list all, user",
args: args{
InstancePermissionV2.WithAuthorizationToken(iamOwnerCtx, integration.UserTypeNoPermission),
&user.ListKeysRequest{Filters: []*user.KeysSearchFilter{onlySinceTestStartFilter}},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
myDataPoint,
},
Pagination: &filter.PaginationResponse{
TotalResult: 1,
AppliedLimit: 100,
},
},
},
{
name: "list by id",
args: args{
iamOwnerCtx,
&user.ListKeysRequest{
Filters: []*user.KeysSearchFilter{
onlySinceTestStartFilter,
{
Filter: &user.KeysSearchFilter_KeyIdFilter{
KeyIdFilter: &filter.IDFilter{Id: otherOrgDataPointExpiringSoon.Id},
},
},
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherOrgDataPointExpiringSoon,
},
Pagination: &filter.PaginationResponse{
TotalResult: 1,
AppliedLimit: 100,
},
},
},
{
name: "list all from other org",
args: args{
iamOwnerCtx,
&user.ListKeysRequest{
Filters: []*user.KeysSearchFilter{
onlySinceTestStartFilter,
{
Filter: &user.KeysSearchFilter_OrganizationIdFilter{
OrganizationIdFilter: &filter.IDFilter{Id: otherOrg.OrganizationId},
},
},
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherOrgDataPointExpiringLate,
otherOrgDataPointExpiringSoon,
},
Pagination: &filter.PaginationResponse{
TotalResult: 2,
AppliedLimit: 100,
},
},
},
{
name: "sort by next expiration dates",
args: args{
iamOwnerCtx,
&user.ListKeysRequest{
Pagination: &filter.PaginationRequest{
Asc: true,
},
SortingColumn: &sortingColumnExpirationDate,
Filters: []*user.KeysSearchFilter{
onlySinceTestStartFilter,
{Filter: &user.KeysSearchFilter_OrganizationIdFilter{OrganizationIdFilter: &filter.IDFilter{Id: otherOrg.OrganizationId}}},
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherOrgDataPointExpiringSoon,
otherOrgDataPointExpiringLate,
},
Pagination: &filter.PaginationResponse{
TotalResult: 2,
AppliedLimit: 100,
},
},
},
{
name: "get page",
args: args{
iamOwnerCtx,
&user.ListKeysRequest{
Pagination: &filter.PaginationRequest{
Offset: 2,
Limit: 2,
Asc: true,
},
Filters: []*user.KeysSearchFilter{
onlySinceTestStartFilter,
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{
otherOrgDataPointExpiringSoon,
otherOrgDataPointExpiringLate,
},
Pagination: &filter.PaginationResponse{
TotalResult: 4,
AppliedLimit: 2,
},
},
},
{
name: "empty list",
args: args{
InstancePermissionV2.WithAuthorizationToken(iamOwnerCtx, integration.UserTypeNoPermission),
&user.ListKeysRequest{
Filters: []*user.KeysSearchFilter{
{
Filter: &user.KeysSearchFilter_KeyIdFilter{
KeyIdFilter: &filter.IDFilter{Id: otherUserDataPoint.Id},
},
},
},
},
},
want: &user.ListKeysResponse{
Result: []*user.Key{},
Pagination: &filter.PaginationResponse{
@@ -584,41 +823,24 @@ func TestServer_ListKeys(t *testing.T) {
},
},
}
t.Run("with permission flag v2", func(t *testing.T) {
setPermissionCheckV2Flag(t, true)
defer setPermissionCheckV2Flag(t, false)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := Client.ListKeys(tt.args.ctx, tt.args.req)
require.NoError(t, err)
assert.Len(t, got.Result, len(tt.want.Result))
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(tt.args.ctx, 20*time.Second)
require.EventuallyWithT(t, func(ttt *assert.CollectT) {
got, err := InstancePermissionV2.Client.UserV2.ListKeys(tt.args.ctx, tt.args.req)
require.NoError(ttt, err)
assert.Len(ttt, got.Result, len(tt.want.Result))
if diff := cmp.Diff(tt.want, got, protocmp.Transform()); diff != "" {
t.Errorf("ListKeys() mismatch (-want +got):\n%s", diff)
ttt.Errorf("ListKeys() mismatch (-want +got):\n%s", diff)
}
})
}
})
t.Run("without permission flag v2", func(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := Client.ListKeys(tt.args.ctx, tt.args.req)
require.NoError(t, err)
assert.Len(t, got.Result, len(tt.want.Result))
// ignore the total result, as this is a known bug with the in-memory permission checks.
// The command can't know how many keys exist in the system if the SQL statement has a limit.
// This is fixed, once the in-memory permission checks are removed with https://github.com/zitadel/zitadel/issues/9188
tt.want.Pagination.TotalResult = got.Pagination.TotalResult
if diff := cmp.Diff(tt.want, got, protocmp.Transform()); diff != "" {
t.Errorf("ListKeys() mismatch (-want +got):\n%s", diff)
}
})
}
})
}, retryDuration, tick, "timeout waiting for expected user result")
})
}
}
func setupKeyDataPoint(t *testing.T, userId, orgId string, expirationDate time.Time) *user.Key {
func setupKeyDataPoint(ctx context.Context, t *testing.T, instance *integration.Instance, userId, orgId string, expirationDate time.Time) *user.Key {
expirationDatePb := timestamppb.New(expirationDate)
newKey, err := Client.AddKey(SystemCTX, &user.AddKeyRequest{
newKey, err := instance.Client.UserV2.AddKey(ctx, &user.AddKeyRequest{
UserId: userId,
ExpirationDate: expirationDatePb,
PublicKey: nil,
@@ -634,18 +856,20 @@ func setupKeyDataPoint(t *testing.T, userId, orgId string, expirationDate time.T
}
}
func awaitKeys(t *testing.T, sinceTestStartFilter *user.KeysSearchFilter, keyIds ...string) {
func awaitKeys(ctx context.Context, t *testing.T, instance *integration.Instance, sinceTestStartFilter *user.KeysSearchFilter, keyIds ...string) {
sortingColumn := user.KeyFieldName_KEY_FIELD_NAME_ID
slices.Sort(keyIds)
retryDuration, tick := integration.WaitForAndTickWithMaxDuration(ctx, time.Minute)
require.EventuallyWithT(t, func(collect *assert.CollectT) {
result, err := Client.ListKeys(SystemCTX, &user.ListKeysRequest{
result, err := instance.Client.UserV2.ListKeys(ctx, &user.ListKeysRequest{
Filters: []*user.KeysSearchFilter{sinceTestStartFilter},
SortingColumn: &sortingColumn,
Pagination: &filter.PaginationRequest{
Asc: true,
},
})
require.NoError(t, err)
require.NoError(collect, err)
if !assert.Len(collect, result.Result, len(keyIds)) {
return
}
@@ -653,5 +877,5 @@ func awaitKeys(t *testing.T, sinceTestStartFilter *user.KeysSearchFilter, keyIds
keyId := keyIds[i]
require.Equal(collect, keyId, result.Result[i].GetId())
}
}, 5*time.Second, time.Second, "key not created in time")
}, retryDuration, tick, "key not created in time")
}