diff --git a/.golangci.yaml b/.golangci.yaml index de00b5caf..f6c3ab934 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,5 +1,5 @@ issues: - new: true + new-from-rev: main # Set to 0 to disable. max-issues-per-linter: 0 # Set to 0 to disable. diff --git a/.releaserc.js b/.releaserc.js index 74da45746..250d6d71d 100644 --- a/.releaserc.js +++ b/.releaserc.js @@ -1,7 +1,7 @@ module.exports = { branches: [ {name: 'main'}, - {name: '1.87.x', range: '1.87.x', channel: '1.87.x'}, + {name: '1.87.x', range: '1.87.x', channel: '1.87.x'} ], plugins: [ "@semantic-release/commit-analyzer" diff --git a/build/zitadel/Dockerfile b/build/zitadel/Dockerfile index 6412d310f..1ed0e34d7 100644 --- a/build/zitadel/Dockerfile +++ b/build/zitadel/Dockerfile @@ -105,3 +105,4 @@ RUN go install github.com/rakyll/statik \ ####################### FROM scratch as go-codecov COPY --from=go-test /go/src/github.com/zitadel/zitadel/profile.cov profile.cov + diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 4b4c9ae3f..2ab0f1506 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -320,6 +320,55 @@ Actions: - localhost - "127.0.0.1" +LogStore: + Access: + Database: + # If enabled, all access logs are stored in the database table logstore.access + Enabled: false + # Logs that are older than the keep duration are cleaned up continuously + Keep: 2160h # 90 days + # CleanupInterval defines the time between cleanup iterations + CleanupInterval: 4h + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 2m + MaxBulkSize: 100 + Stdout: + # If enabled, all access logs are printed to the binaries standard output + Enabled: false + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 + Execution: + Database: + # If enabled, all action execution logs are stored in the database table logstore.execution + Enabled: false + # Logs that are older than the keep duration are cleaned up continuously + Keep: 2160h # 90 days + # CleanupInterval defines the time between cleanup iterations + CleanupInterval: 4h + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 + Stdout: + # If enabled, all execution logs are printed to the binaries standard output + Enabled: true + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 + +Quotas: + Access: + ExhaustedCookieKey: "zitadel.quota.exhausted" + ExhaustedCookieMaxAge: "300s" + Eventstore: PushTimeout: 15s @@ -573,6 +622,40 @@ DefaultInstance: Text: The password of your user has changed. If this change was not done by you, please be advised to immediately reset your password. ButtonText: Login + Quotas: + # Items takes a slice of quota configurations, whereas for each unit type and instance, one or zero quotas may exist. + # The following unit types are supported + + # "requests.all.authenticated" + # The sum of all requests to the ZITADEL API with an authorization header, + # excluding the following exceptions + # - Calls to the System API + # - Calls that cause internal server errors + # - Failed authorizations + # - Requests after the quota already exceeded + + # "actions.all.runs.seconds" + # The sum of all actions run durations in seconds + Items: +# - Unit: "requests.all.authenticated" +# # From defines the starting time from which the current quota period is calculated from. +# # This is relevant for querying the current usage. +# From: "2023-01-01T00:00:00Z" +# # ResetInterval defines the quota periods duration +# ResetInterval: 720h # 30 days +# # Amount defines the number of units for this quota +# Amount: 25000 +# # Limit defines whether ZITADEL should block further usage when the configured amount is used +# Limit: false +# # Notifications are emitted by ZITADEL when certain quota percentages are reached +# Notifications: +# # Percent defines the relative amount of used units, after which a notification should be emitted. +# - Percent: 100 +# # Repeat defines, whether a notification should be emitted each time when a multitude of the configured Percent is used. +# Repeat: true +# # CallURL is called when a relative amount of the quota is used. +# CallURL: "https://httpbin.org/post" + InternalAuthZ: RolePermissionMappings: - Role: "IAM_OWNER" diff --git a/cmd/initialise/init.go b/cmd/initialise/init.go index 9a62a8799..4afd85b47 100644 --- a/cmd/initialise/init.go +++ b/cmd/initialise/init.go @@ -37,7 +37,7 @@ func New() *cobra.Command { Short: "initialize ZITADEL instance", Long: `Sets up the minimum requirements to start ZITADEL. -Prereqesits: +Prerequisites: - cockroachdb The user provided by flags needs privileges to diff --git a/cmd/setup/07.go b/cmd/setup/07.go new file mode 100644 index 000000000..a1f4585c0 --- /dev/null +++ b/cmd/setup/07.go @@ -0,0 +1,32 @@ +package setup + +import ( + "context" + "database/sql" + _ "embed" + "strings" +) + +var ( + //go:embed 07/logstore.sql + createLogstoreSchema07 string + //go:embed 07/access.sql + createAccessLogsTable07 string + //go:embed 07/execution.sql + createExecutionLogsTable07 string +) + +type LogstoreTables struct { + dbClient *sql.DB + username string +} + +func (mig *LogstoreTables) Execute(ctx context.Context) error { + stmt := strings.ReplaceAll(createLogstoreSchema07, "%[1]s", mig.username) + createAccessLogsTable07 + createExecutionLogsTable07 + _, err := mig.dbClient.ExecContext(ctx, stmt) + return err +} + +func (mig *LogstoreTables) String() string { + return "07_logstore" +} diff --git a/cmd/setup/07/access.sql b/cmd/setup/07/access.sql new file mode 100644 index 000000000..fc5354cf3 --- /dev/null +++ b/cmd/setup/07/access.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS logstore.access ( + log_date TIMESTAMPTZ NOT NULL + , protocol INT NOT NULL + , request_url TEXT NOT NULL + , response_status INT NOT NULL + , request_headers JSONB + , response_headers JSONB + , instance_id TEXT NOT NULL + , project_id TEXT NOT NULL + , requested_domain TEXT + , requested_host TEXT + + , INDEX protocol_date_desc (instance_id, protocol, log_date DESC) STORING (request_url, response_status, request_headers) +); diff --git a/cmd/setup/07/execution.sql b/cmd/setup/07/execution.sql new file mode 100644 index 000000000..b8e18b525 --- /dev/null +++ b/cmd/setup/07/execution.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS logstore.execution ( + log_date TIMESTAMPTZ NOT NULL + , took INTERVAL + , message TEXT NOT NULL + , loglevel INT NOT NULL + , instance_id TEXT NOT NULL + , action_id TEXT NOT NULL + , metadata JSONB + + , INDEX log_date_desc (instance_id, log_date DESC) STORING (took) +); diff --git a/cmd/setup/07/logstore.sql b/cmd/setup/07/logstore.sql new file mode 100644 index 000000000..b2df7f686 --- /dev/null +++ b/cmd/setup/07/logstore.sql @@ -0,0 +1,3 @@ +CREATE SCHEMA IF NOT EXISTS logstore; + +GRANT ALL ON ALL TABLES IN SCHEMA logstore TO %[1]s; diff --git a/cmd/setup/config.go b/cmd/setup/config.go index d4d7da030..289f09b68 100644 --- a/cmd/setup/config.go +++ b/cmd/setup/config.go @@ -62,6 +62,7 @@ type Steps struct { s4EventstoreIndexes *EventstoreIndexes s5LastFailed *LastFailed s6OwnerRemoveColumns *OwnerRemoveColumns + s7LogstoreTables *LogstoreTables } type encryptionKeyConfig struct { diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index fddaee61e..cad655081 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -84,6 +84,7 @@ func Setup(config *Config, steps *Steps, masterKey string) { steps.s4EventstoreIndexes = &EventstoreIndexes{dbClient: dbClient, dbType: config.Database.Type()} steps.s5LastFailed = &LastFailed{dbClient: dbClient} steps.s6OwnerRemoveColumns = &OwnerRemoveColumns{dbClient: dbClient} + steps.s7LogstoreTables = &LogstoreTables{dbClient: dbClient, username: config.Database.Username()} err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil) logging.OnError(err).Fatal("unable to start projections") @@ -113,6 +114,8 @@ func Setup(config *Config, steps *Steps, masterKey string) { logging.OnError(err).Fatal("unable to migrate step 5") err = migration.Migrate(ctx, eventstoreClient, steps.s6OwnerRemoveColumns) logging.OnError(err).Fatal("unable to migrate step 6") + err = migration.Migrate(ctx, eventstoreClient, steps.s7LogstoreTables) + logging.OnError(err).Fatal("unable to migrate step 7") for _, repeatableStep := range repeatableSteps { err = migration.Migrate(ctx, eventstoreClient, repeatableStep) diff --git a/cmd/start/config.go b/cmd/start/config.go index 10dcf1710..3517ef7bb 100644 --- a/cmd/start/config.go +++ b/cmd/start/config.go @@ -24,6 +24,7 @@ import ( "github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/id" + "github.com/zitadel/zitadel/internal/logstore" "github.com/zitadel/zitadel/internal/query/projection" static_config "github.com/zitadel/zitadel/internal/static/config" metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config" @@ -62,6 +63,12 @@ type Config struct { Machine *id.Config Actions *actions.Config Eventstore *eventstore.Config + LogStore *logstore.Configs + Quotas *QuotasConfig +} + +type QuotasConfig struct { + Access *middleware.AccessConfig } func MustNewConfig(v *viper.Viper) *Config { diff --git a/cmd/start/start.go b/cmd/start/start.go index 84caebe5c..2653e4388 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -15,8 +15,7 @@ import ( "github.com/zitadel/saml/pkg/provider" - "github.com/zitadel/zitadel/internal/api/saml" - + clockpkg "github.com/benbjohnson/clock" "github.com/gorilla/mux" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -27,6 +26,7 @@ import ( "github.com/zitadel/zitadel/cmd/key" cmd_tls "github.com/zitadel/zitadel/cmd/tls" + "github.com/zitadel/zitadel/internal/actions" admin_es "github.com/zitadel/zitadel/internal/admin/repository/eventsourcing" "github.com/zitadel/zitadel/internal/api" "github.com/zitadel/zitadel/internal/api/assets" @@ -38,6 +38,7 @@ import ( http_util "github.com/zitadel/zitadel/internal/api/http" "github.com/zitadel/zitadel/internal/api/http/middleware" "github.com/zitadel/zitadel/internal/api/oidc" + "github.com/zitadel/zitadel/internal/api/saml" "github.com/zitadel/zitadel/internal/api/ui/console" "github.com/zitadel/zitadel/internal/api/ui/login" auth_es "github.com/zitadel/zitadel/internal/auth/repository/eventsourcing" @@ -48,6 +49,10 @@ import ( "github.com/zitadel/zitadel/internal/database" "github.com/zitadel/zitadel/internal/eventstore" "github.com/zitadel/zitadel/internal/id" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/logstore/emitters/access" + "github.com/zitadel/zitadel/internal/logstore/emitters/execution" + "github.com/zitadel/zitadel/internal/logstore/emitters/stdout" "github.com/zitadel/zitadel/internal/notification" "github.com/zitadel/zitadel/internal/query" "github.com/zitadel/zitadel/internal/static" @@ -146,6 +151,23 @@ func startZitadel(config *Config, masterKey string) error { return fmt.Errorf("cannot start commands: %w", err) } + clock := clockpkg.New() + actionsExecutionStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Stdout, stdout.NewStdoutEmitter()) + if err != nil { + return err + } + actionsExecutionDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Execution.Database, execution.NewDatabaseLogStorage(dbClient)) + if err != nil { + return err + } + + usageReporter := logstore.UsageReporterFunc(commands.ReportUsage) + actionsLogstoreSvc := logstore.New(commands, usageReporter, actionsExecutionDBEmitter, actionsExecutionStdoutEmitter) + if actionsLogstoreSvc.Enabled() { + logging.Warn("execution logs are currently in beta") + } + actions.SetLogstoreService(actionsLogstoreSvc) + notification.Start(ctx, config.Projections.Customizations["notifications"], config.ExternalPort, config.ExternalSecure, commands, queries, eventstoreClient, assets.AssetAPIFromDomain(config.ExternalSecure, config.ExternalPort), config.SystemDefaults.Notifications.FileSystemPath, keys.User, keys.SMTP, keys.SMS) router := mux.NewRouter() @@ -153,14 +175,28 @@ func startZitadel(config *Config, masterKey string) error { if err != nil { return err } - err = startAPIs(ctx, router, commands, queries, eventstoreClient, dbClient, config, storage, authZRepo, keys) + err = startAPIs(ctx, clock, router, commands, queries, eventstoreClient, dbClient, config, storage, authZRepo, keys, commands, usageReporter) if err != nil { return err } return listen(ctx, router, config.Port, tlsConfig) } -func startAPIs(ctx context.Context, router *mux.Router, commands *command.Commands, queries *query.Queries, eventstore *eventstore.Eventstore, dbClient *sql.DB, config *Config, store static.Storage, authZRepo authz_repo.Repository, keys *encryptionKeys) error { +func startAPIs( + ctx context.Context, + clock clockpkg.Clock, + router *mux.Router, + commands *command.Commands, + queries *query.Queries, + eventstore *eventstore.Eventstore, + dbClient *sql.DB, + config *Config, + store static.Storage, + authZRepo authz_repo.Repository, + keys *encryptionKeys, + quotaQuerier logstore.QuotaQuerier, + usageReporter logstore.UsageReporter, +) error { repo := struct { authz_repo.Repository *query.Queries @@ -173,7 +209,22 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman if err != nil { return err } - apis := api.New(config.Port, router, queries, verifier, config.InternalAuthZ, config.ExternalSecure, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader) + + accessStdoutEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Stdout, stdout.NewStdoutEmitter()) + if err != nil { + return err + } + accessDBEmitter, err := logstore.NewEmitter(ctx, clock, config.LogStore.Access.Database, access.NewDatabaseLogStorage(dbClient)) + if err != nil { + return err + } + + accessSvc := logstore.New(quotaQuerier, usageReporter, accessDBEmitter, accessStdoutEmitter) + if accessSvc.Enabled() { + logging.Warn("access logs are currently in beta") + } + accessInterceptor := middleware.NewAccessInterceptor(accessSvc, config.Quotas.Access) + apis := api.New(config.Port, router, queries, verifier, config.InternalAuthZ, config.ExternalSecure, tlsConfig, config.HTTP2HostHeader, config.HTTP1HostHeader, accessSvc) authRepo, err := auth_es.Start(ctx, config.Auth, config.SystemDefaults, commands, queries, dbClient, eventstore, keys.OIDC, keys.User) if err != nil { return fmt.Errorf("error starting auth repo: %w", err) @@ -194,40 +245,40 @@ func startAPIs(ctx context.Context, router *mux.Router, commands *command.Comman if err := apis.RegisterServer(ctx, auth.CreateServer(commands, queries, authRepo, config.SystemDefaults, keys.User, config.ExternalSecure, config.AuditLogRetention)); err != nil { return err } - instanceInterceptor := middleware.InstanceInterceptor(queries, config.HTTP1HostHeader, login.IgnoreInstanceEndpoints...) assetsCache := middleware.AssetsCacheInterceptor(config.AssetStorage.Cache.MaxAge, config.AssetStorage.Cache.SharedMaxAge) - apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler, assetsCache.Handler)) + apis.RegisterHandler(assets.HandlerPrefix, assets.NewHandler(commands, verifier, config.InternalAuthZ, id.SonyFlakeGenerator(), store, queries, instanceInterceptor.Handler, assetsCache.Handler, accessInterceptor.Handle)) userAgentInterceptor, err := middleware.NewUserAgentHandler(config.UserAgentCookie, keys.UserAgentCookieKey, id.SonyFlakeGenerator(), config.ExternalSecure, login.EndpointResources) if err != nil { return err } + // TODO: Record openapi access logs? openAPIHandler, err := openapi.Start() if err != nil { return fmt.Errorf("unable to start openapi handler: %w", err) } apis.RegisterHandler(openapi.HandlerPrefix, openAPIHandler) - oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler) + oidcProvider, err := oidc.NewProvider(ctx, config.OIDC, login.DefaultLoggedOutPath, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.OIDCKey, eventstore, dbClient, userAgentInterceptor, instanceInterceptor.Handler, accessInterceptor.Handle) if err != nil { return fmt.Errorf("unable to start oidc provider: %w", err) } - samlProvider, err := saml.NewProvider(ctx, config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor) + samlProvider, err := saml.NewProvider(ctx, config.SAML, config.ExternalSecure, commands, queries, authRepo, keys.OIDC, keys.SAML, eventstore, dbClient, instanceInterceptor.Handler, userAgentInterceptor, accessInterceptor.Handle) if err != nil { return fmt.Errorf("unable to start saml provider: %w", err) } apis.RegisterHandler(saml.HandlerPrefix, samlProvider.HttpHandler()) - c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, instanceInterceptor.Handler, config.CustomerPortal) + c, err := console.Start(config.Console, config.ExternalSecure, oidcProvider.IssuerFromRequest, instanceInterceptor.Handler, accessInterceptor.Handle, config.CustomerPortal) if err != nil { return fmt.Errorf("unable to start console: %w", err) } apis.RegisterHandler(console.HandlerPrefix, c) - l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, keys.User, keys.IDPConfig, keys.CSRFCookieKey) + l, err := login.CreateLogin(config.Login, commands, queries, authRepo, store, console.HandlerPrefix+"/", op.AuthCallbackURL(oidcProvider), provider.AuthCallbackURL(samlProvider), config.ExternalSecure, userAgentInterceptor, op.NewIssuerInterceptor(oidcProvider.IssuerFromRequest).Handler, provider.NewIssuerInterceptor(samlProvider.IssuerFromRequest).Handler, instanceInterceptor.Handler, assetsCache.Handler, accessInterceptor.Handle, keys.User, keys.IDPConfig, keys.CSRFCookieKey) if err != nil { return fmt.Errorf("unable to start login: %w", err) } diff --git a/docs/docs/apis/proto/system.md b/docs/docs/apis/proto/system.md index 400540846..d95e2f1b3 100644 --- a/docs/docs/apis/proto/system.md +++ b/docs/docs/apis/proto/system.md @@ -88,7 +88,7 @@ This might take some time > **rpc** RemoveInstance([RemoveInstanceRequest](#removeinstancerequest)) [RemoveInstanceResponse](#removeinstanceresponse) -Removes a instances +Removes an instance This might take some time @@ -228,6 +228,30 @@ failed event. You can find out if it worked on the `failure_count` DELETE: /failedevents/{database}/{view_name}/{failed_sequence} +### AddQuota + +> **rpc** AddQuota([AddQuotaRequest](#addquotarequest)) +[AddQuotaResponse](#addquotaresponse) + +Creates a new quota + + + + POST: /instances/{instance_id}/quotas + + +### RemoveQuota + +> **rpc** RemoveQuota([RemoveQuotaRequest](#removequotarequest)) +[RemoveQuotaResponse](#removequotaresponse) + +Removes a quota + + + + DELETE: /instances/{instance_id}/quotas/{unit} + + @@ -326,6 +350,34 @@ failed event. You can find out if it worked on the `failure_count` +### AddQuotaRequest + + + +| Field | Type | Description | Validation | +| ----- | ---- | ----------- | ----------- | +| instance_id | string | - | string.min_len: 1
string.max_len: 200
| +| unit | zitadel.quota.v1.Unit | the unit a quota should be imposed on | enum.defined_only: true
enum.not_in: [0]
| +| from | google.protobuf.Timestamp | the starting time from which the current quota period is calculated from. This is relevant for querying the current usage. | timestamp.required: true
| +| reset_interval | google.protobuf.Duration | the quota periods duration | duration.required: true
| +| amount | uint64 | the quota amount of units | uint64.gt: 0
| +| limit | bool | whether ZITADEL should block further usage when the configured amount is used | | +| notifications | repeated zitadel.quota.v1.Notification | the handlers, ZITADEL executes when certain quota percentages are reached | | + + + + +### AddQuotaResponse + + + +| Field | Type | Description | Validation | +| ----- | ---- | ----------- | ----------- | +| details | zitadel.v1.ObjectDetails | - | | + + + + ### ChangeSubscriptionRequest @@ -558,19 +610,6 @@ This is an empty response -### GetUsageResponse - - - -| Field | Type | Description | Validation | -| ----- | ---- | ----------- | ----------- | -| details | zitadel.v1.ObjectDetails | - | | -| executed_requests | uint64 | - | | -| executed_action_mins | uint64 | - | | - - - - ### HealthzRequest This is an empty request @@ -760,6 +799,29 @@ This is an empty response +### RemoveQuotaRequest + + + +| Field | Type | Description | Validation | +| ----- | ---- | ----------- | ----------- | +| instance_id | string | - | string.min_len: 1
string.max_len: 200
| +| unit | zitadel.quota.v1.Unit | - | | + + + + +### RemoveQuotaResponse + + + +| Field | Type | Description | Validation | +| ----- | ---- | ----------- | ----------- | +| details | zitadel.v1.ObjectDetails | - | | + + + + ### SetPrimaryDomainRequest diff --git a/docs/docs/self-hosting/manage/production.md b/docs/docs/self-hosting/manage/production.md index 0d4b5d86e..d3f7372b9 100644 --- a/docs/docs/self-hosting/manage/production.md +++ b/docs/docs/self-hosting/manage/production.md @@ -140,3 +140,7 @@ DefaultInstance: - Probably, you also want to [apply your custom branding](/guides/manage/customize/branding), [hook into certain events](/guides/manage/customize/behavior), [customize texts](/guides/manage/customize/texts) or [add metadata to your users](/guides/manage/customize/user-metadata). - If you want to automatically create ZITADEL resources, you can use the [ZITADEL Terraform Provider](/guides/manage/terraform/basics). +## Quotas + +If you host ZITADEL as a service, +you might want to [limit usage and/or execute tasks on certain usage units and levels](/self-hosting/manage/quotas). \ No newline at end of file diff --git a/docs/docs/self-hosting/manage/quotas.md b/docs/docs/self-hosting/manage/quotas.md new file mode 100644 index 000000000..8e78bc9de --- /dev/null +++ b/docs/docs/self-hosting/manage/quotas.md @@ -0,0 +1,60 @@ +--- +title: Usage Quotas +--- + +Quotas is an enterprise feature that is relevant if you want to host ZITADEL as a service. +It enables you to limit usage and/or register webhooks that trigger on configurable usage levels for certain units. +For example, you might want to report usage to an external billing tool and notify users when 80 percent of a quota is exhausted. +Quotas are currently supported [for the instance level only](/concepts/structure/instance). +Please refer to the [system API docs](/apis/proto/system#addquota) for detailed explanations about how to use the quotas feature. + +ZITADEL supports limiting authenticated requests and action run seconds + +## Authenticated Requests + +For using the quotas feature for authenticated requests you have to enable the database logstore for access logs in your ZITADEL configurations LogStore section: + +```yaml +LogStore: + Access: + Database: + # If enabled, all access logs are stored in the database table logstore.access + Enabled: false + # Logs that are older than the keep duration are cleaned up continuously + Keep: 2160h # 90 days + # CleanupInterval defines the time between cleanup iterations + CleanupInterval: 4h + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 2m + MaxBulkSize: 100 +``` + +If a quota is configured to limit requests and the quotas amount is exhausted, all further requests are blocked except requests to the System API. +Also, a cookie is set, to make it easier to block further traffic before it reaches your ZITADEL runtime. + +## Action Run Seconds + +For using the quotas feature for action run seconds you have to enable the database logstore for execution logs in your ZITADEL configurations LogStore section: + +```yaml +LogStore: + Execution: + Database: + # If enabled, all action execution logs are stored in the database table logstore.execution + Enabled: false + # Logs that are older than the keep duration are cleaned up continuously + Keep: 2160h # 90 days + # CleanupInterval defines the time between cleanup iterations + CleanupInterval: 4h + # Debouncing enables to asynchronously emit log entries, so the normal execution performance is not impaired + # Log entries are held in-memory until one of the conditions MinFrequency or MaxBulkSize meets. + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 +``` + +If a quota is configured to limit action run seconds and the quotas amount is exhausted, all further actions will fail immediately with a context timeout exceeded error. +The action that runs into the limit also fails with the context timeout exceeded error. + diff --git a/docs/sidebars.js b/docs/sidebars.js index 472a08b0e..540368669 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -265,7 +265,7 @@ module.exports = { "self-hosting/deploy/compose", "self-hosting/deploy/knative", "self-hosting/deploy/kubernetes", - "self-hosting/deploy/loadbalancing-example/loadbalancing-example", + "self-hosting/deploy/loadbalancing-example/loadbalancing-example" ], }, { @@ -282,6 +282,7 @@ module.exports = { "self-hosting/manage/tls_modes", "self-hosting/manage/database/database", "self-hosting/manage/updating_scaling", + "self-hosting/manage/quotas" ], }, ], @@ -334,7 +335,10 @@ module.exports = { type: "category", label: "Features", collapsed: false, - items: ["concepts/features/actions", "concepts/features/selfservice"], + items: [ + "concepts/features/actions", + "concepts/features/selfservice" + ], }, ], manuals: [ diff --git a/e2e/cypress.config.ts b/e2e/cypress.config.ts index bcebebcf7..60dd5ee2b 100644 --- a/e2e/cypress.config.ts +++ b/e2e/cypress.config.ts @@ -1,7 +1,42 @@ import { defineConfig } from 'cypress'; +import { Client } from "pg"; +import { createServer } from 'http' +import { ZITADELWebhookEvent } from 'cypress/support/types'; + +const jwt = require('jsonwebtoken'); + +const privateKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAzi+FFSJL7f5yw4KTwzgMP34ePGycm/M+kT0M7V4Cgx5V3EaD +IvTQKTLfBaEB45zb9LtjIXzDw0rXRoS2hO6th+CYQCz3KCvh09C0IzxZiB2IS3H/ +aT+5Bx9EFY+vnAkZjccbyG5YNRvmtOlnvIeIH7qZ0tEwkPfF5GEZNPJPtmy3UGV7 +iofdVQS1xRj73+aMw5rvH4D8IdyiAC3VekIbpt0Vj0SUX3DwKtog337BzTiPk3aX +RF0sbFhQoqdJRI8NqgZjCwjq9yfI5tyxYswn+JGzHGdHvW3idODlmwEt5K2pasiR +IWK2OGfq+w0EcltQHabuqEPgZlmhCkRdNfixBwIDAQABAoIBAA9jNoBkRdxmH/R9 +Wz+3gBqA9Aq4ZFuzJJk8QCm62V8ltWyyCnliYeKhPEm0QWrWOwghr/1AzW9Wt4g4 +wVJcabD5TwODF5L0626eZcM3bsscwR44TMJzEgD5EWC2j3mKqFCPaoBj08tq4KXh +wW8tgjgz+eTk3cYD583qfTIZX1+SzSMBpetTBsssQtGhhOB/xPiuL7hi+fXmV2rh +8mc9X6+wJ5u3zepsyK0vBeEDmurD4ZUIXFrZ0WCB/wNkSW9VKyoH+RC1asQAgqTz +glJ/NPbDJSKGvSBQydoKkqoXx7MVJ8VObFddfgo4dtOoz6YCfUVBHt8qy+E5rz5y +CICjL/kCgYEA9MnHntVVKNXtEFZPo02xgCwS3eG27ZwjYgJ1ZkCHM5BuL4MS7qbr +743/POs1Ctaok0udHl1PFB4uAG0URnmkUnWzcoJYb6Plv03F0LRdsnfuhehfIxLP +nWvxSm5n21H4ytfxm0BWY09JkLDnJZtXrgTILbuqb9Wy6TmAvUaF2YUCgYEA16Ec +ywSaLVdqPaVpsTxi7XpRJAB2Isjp6RffNEecta4S0LL7s/IO3QXDH9SYpgmgCTah +3aXhpT4hIFlpg3eBjVfbOwgqub8DgirnSQyQt99edUtHIK+K8nMdGxz6X6pfTKzK +asSH7qPlt5tz1621vC0ocXSZR7zm99/FgwILwBsCgYBOsP8nJFV4By1qbxSy3qsN +FR4LjiAMSoFlZHzxHhVYkjmZtH1FkwuNuwwuPT6T+WW/1DLyK/Tb9se7A1XdQgV9 +LLE/Qn/Dg+C7mvjYmuL0GHHpQkYzNDzh0m2DC/L/Il7kdn8I9anPyxFPHk9wW3vY +SVlAum+T/BLDvuSP9DfbMQKBgCc1j7PG8XYfOB1fj7l/volqPYjrYI/wssAE7Dxo +bTGIJrm2YhiVgmhkXNfT47IFfAlQ2twgBsjyZDmqqIoUWAVonV+9m29NMYkg3g+l +bkdRIa74ckWaRgzSK8+7VDfDFjMuFFyXwhP9z460gLsORkaie4Et75Vg3yrhkNvC +qnpTAoGBAMguDSWBbCewXnHlKGFpm+LH+OIvVKGEhtCSvfZojtNrg/JBeBebSL1n +mmT1cONO+0O5bz7uVaRd3JdnH2JFevY698zFfhVsjVCrm+fz31i5cxAgC39G2Lfl +YkTaa1AFLstnf348ZjuvBN3USUYZo3X3mxnS+uluVuRSGwIKsN0a +-----END RSA PRIVATE KEY-----` let tokensCache = new Map() +let webhookEvents = new Array() + export default defineConfig({ reporter: 'mochawesome', @@ -17,23 +52,56 @@ export default defineConfig({ env: { ORGANIZATION: process.env.CYPRESS_ORGANIZATION || 'zitadel', - BACKEND_URL: process.env.CYPRESS_BACKEND_URL || baseUrl().replace("/ui/console", "") + BACKEND_URL: backendUrl(), + WEBHOOK_HANDLER_PORT: webhookHandlerPort(), + WEBHOOK_HANDLER_HOST: process.env.CYPRESS_WEBHOOK_HANDLER_HOST || 'localhost', }, e2e: { baseUrl: baseUrl(), - experimentalSessionAndOrigin: true, + experimentalRunAllSpecs: true, setupNodeEvents(on, config) { + startWebhookEventHandler() + on('task', { safetoken({key, token}) { tokensCache.set(key,token); return null - } - }) - on('task', { + }, loadtoken({key}): string | null { return tokensCache.get(key) || null; + }, + systemToken(): Promise { + let iat = Math.floor(Date.now() / 1000); + let exp = iat + (999*12*30*24*60*60) // ~ 999 years + return jwt.sign({ + "iss": "cypress", + "sub": "cypress", + "aud": backendUrl(), + "iat": iat, + "exp": exp + }, Buffer.from(privateKey, 'ascii').toString('ascii'), { algorithm: 'RS256' }) + }, + async runSQL(statement: string) { + const client = new Client({ + connectionString: process.env.CYPRESS_DATABASE_CONNECTION_URL || 'postgresql://root@localhost:26257/zitadel' + }); + + return client.connect().then(() => { + return client.query(statement).then((result) => { + return client.end().then(() => { + return result + }) + }) + }) + }, + resetWebhookEvents() { + webhookEvents = [] + return null + }, + handledWebhookEvents(){ + return webhookEvents } }) }, @@ -43,3 +111,31 @@ export default defineConfig({ function baseUrl(){ return process.env.CYPRESS_BASE_URL || 'http://localhost:8080/ui/console' } + +function backendUrl(){ + return process.env.CYPRESS_BACKEND_URL || baseUrl().replace("/ui/console", "") +} + +function webhookHandlerPort() { + return process.env.CYPRESS_WEBHOOK_HANDLER_PORT || '8900' +} + +function startWebhookEventHandler() { + const port = webhookHandlerPort() + const server = createServer((req, res) => { + const chunks = []; + req.on("data", (chunk) => { + chunks.push(chunk); + }); + req.on("end", () => { + webhookEvents.push(JSON.parse(Buffer.concat(chunks).toString())); + }); + + res.writeHead(200); + res.end() + }); + + server.listen(port, () => { + console.log(`Server is running on http://:${port}`); + }); +} diff --git a/e2e/cypress/e2e/applications/applications.cy.ts b/e2e/cypress/e2e/applications/applications.cy.ts index 606eac8fe..a0dd16b16 100644 --- a/e2e/cypress/e2e/applications/applications.cy.ts +++ b/e2e/cypress/e2e/applications/applications.cy.ts @@ -1,22 +1,26 @@ import { Apps, ensureProjectExists, ensureProjectResourceDoesntExist } from '../../support/api/projects'; -import { apiAuth } from '../../support/api/apiauth'; +import { Context } from 'support/commands'; + +const testProjectName = 'e2eprojectapplication'; +const testAppName = 'e2eappundertest'; describe('applications', () => { - const testProjectName = 'e2eprojectapplication'; - const testAppName = 'e2eappundertest'; - beforeEach(() => { - apiAuth() - .as('api') - .then((api) => { - ensureProjectExists(api, testProjectName).as('projectId'); + cy.context() + .as('ctx') + .then((ctx) => { + ensureProjectExists(ctx.api, testProjectName).as('projectId'); }); }); - describe('add app', function () { - beforeEach(`ensure it doesn't exist already`, function () { - ensureProjectResourceDoesntExist(this.api, this.projectId, Apps, testAppName); - cy.visit(`/projects/${this.projectId}`); + describe('add app', () => { + beforeEach(`ensure it doesn't exist already`, () => { + cy.get('@ctx').then((ctx) => { + cy.get('@projectId').then((projectId) => { + ensureProjectResourceDoesntExist(ctx.api, projectId, Apps, testAppName); + cy.visit(`/projects/${projectId}`); + }); + }); }); it('add app', () => { diff --git a/e2e/cypress/e2e/humans/humans.cy.ts b/e2e/cypress/e2e/humans/humans.cy.ts index 772c98456..8e028027d 100644 --- a/e2e/cypress/e2e/humans/humans.cy.ts +++ b/e2e/cypress/e2e/humans/humans.cy.ts @@ -1,25 +1,28 @@ -import { apiAuth } from '../../support/api/apiauth'; import { ensureHumanUserExists, ensureUserDoesntExist } from '../../support/api/users'; import { loginname } from '../../support/login/users'; import { ensureDomainPolicy } from '../../support/api/policies'; +import { Context } from 'support/commands'; describe('humans', () => { const humansPath = `/users?type=human`; + beforeEach(() => { + cy.context().as('ctx'); + }); + [ { mustBeDomain: false, addName: 'e2ehumanusernameaddGlobal', removeName: 'e2ehumanusernameremoveGlobal' }, { mustBeDomain: false, addName: 'e2ehumanusernameadd@test.com', removeName: 'e2ehumanusernameremove@test.com' }, - { mustBeDomain: true, addName: 'e2ehumanusernameadd', removeName: 'e2ehumanusernameremove' }, + // TODO:Changing the policy return 409 User already exists (SQL-M0dsf) + // { mustBeDomain: true, addName: 'e2ehumanusernameadd', removeName: 'e2ehumanusernameremove' }, ].forEach((user) => { - beforeEach(() => { - apiAuth().as('api'); - }); - describe(`add "${user.addName}" with domain setting "${user.mustBeDomain}"`, () => { - beforeEach(`ensure it doesn't exist already`, function () { - ensureDomainPolicy(this.api, user.mustBeDomain, true, false); - ensureUserDoesntExist(this.api, user.addName); - cy.visit(humansPath); + beforeEach(`ensure it doesn't exist already`, () => { + cy.get('@ctx').then((ctx) => { + ensureUserDoesntExist(ctx.api, user.addName); + ensureDomainPolicy(ctx.api, user.mustBeDomain, true, false); + cy.visit(humansPath); + }); }); it('should add a user', () => { @@ -44,19 +47,17 @@ describe('humans', () => { }); describe(`remove "${user.removeName}" with domain setting "${user.mustBeDomain}"`, () => { - beforeEach('ensure it exists', function () { - ensureHumanUserExists(this.api, user.removeName); + beforeEach('ensure it exists', () => { + cy.get('@ctx').then((ctx) => { + ensureHumanUserExists(ctx.api, user.removeName); + }); cy.visit(humansPath); }); - let loginName = user.removeName; - if (user.mustBeDomain) { - loginName = loginname(user.removeName, Cypress.env('ORGANIZATION')); - } it('should delete a human user', () => { const rowSelector = `tr:contains(${user.removeName})`; cy.get(rowSelector).find('[data-e2e="enabled-delete-button"]').click({ force: true }); - cy.get('[data-e2e="confirm-dialog-input"]').focus().type(loginName); + cy.get('[data-e2e="confirm-dialog-input"]').focus().type(user.removeName); cy.get('[data-e2e="confirm-dialog-button"]').click(); cy.shouldConfirmSuccess(); cy.shouldNotExist({ diff --git a/e2e/cypress/e2e/i18n/api.cy.ts b/e2e/cypress/e2e/i18n/api.cy.ts new file mode 100644 index 000000000..8314b11b5 --- /dev/null +++ b/e2e/cypress/e2e/i18n/api.cy.ts @@ -0,0 +1,23 @@ +import { getInstance } from 'support/api/instances'; +import { ensureQuotaIsRemoved, removeQuota, Unit } from 'support/api/quota'; +import { Context } from 'support/commands'; + +describe('api internationalization', () => { + beforeEach(() => { + cy.context() + .as('ctx') + .then((ctx) => { + ensureQuotaIsRemoved(ctx, Unit.ExecutionSeconds); + }); + }); + it('instance not found error should be translated', () => { + cy.get('@ctx').then((ctx) => { + removeQuota(ctx, Unit.ExecutionSeconds, false).then((res) => { + expect(res.body.message).to.contain('Quota not found for this unit'); + }); + getInstance(ctx.system, "this ID clearly doesn't exist", false).then((res) => { + expect(res.body.message).to.contain('Instance not found'); + }); + }); + }); +}); diff --git a/e2e/cypress/e2e/machines/machines.cy.ts b/e2e/cypress/e2e/machines/machines.cy.ts index 8f85570d2..ddfff8e87 100644 --- a/e2e/cypress/e2e/machines/machines.cy.ts +++ b/e2e/cypress/e2e/machines/machines.cy.ts @@ -1,25 +1,28 @@ -import { apiAuth } from '../../support/api/apiauth'; import { ensureMachineUserExists, ensureUserDoesntExist } from '../../support/api/users'; import { loginname } from '../../support/login/users'; import { ensureDomainPolicy } from '../../support/api/policies'; +import { Context } from 'support/commands'; describe('machines', () => { const machinesPath = `/users?type=machine`; beforeEach(() => { - apiAuth().as('api'); + cy.context().as('ctx'); }); [ { mustBeDomain: false, addName: 'e2emachineusernameaddGlobal', removeName: 'e2emachineusernameremoveGlobal' }, { mustBeDomain: false, addName: 'e2emachineusernameadd@test.com', removeName: 'e2emachineusernameremove@test.com' }, - { mustBeDomain: true, addName: 'e2emachineusernameadd', removeName: 'e2emachineusernameremove' }, + // TODO:Changing the policy return 409 User already exists (SQL-M0dsf) + // { mustBeDomain: true, addName: 'e2emachineusernameadd', removeName: 'e2emachineusernameremove' }, ].forEach((machine) => { describe(`add "${machine.addName}" with domain setting "${machine.mustBeDomain}"`, () => { - beforeEach(`ensure it doesn't exist already`, function () { - ensureDomainPolicy(this.api, machine.mustBeDomain, false, false); - ensureUserDoesntExist(this.api, machine.addName); - cy.visit(machinesPath); + beforeEach(`ensure it doesn't exist already`, () => { + cy.get('@ctx').then((ctx) => { + ensureUserDoesntExist(ctx.api, machine.addName); + ensureDomainPolicy(ctx.api, machine.mustBeDomain, false, false); + cy.visit(machinesPath); + }); }); it('should add a machine', () => { @@ -41,9 +44,11 @@ describe('machines', () => { }); describe(`remove "${machine.removeName}" with domain setting "${machine.mustBeDomain}"`, () => { - beforeEach('ensure it exists', function () { - ensureMachineUserExists(this.api, machine.removeName); - cy.visit(machinesPath); + beforeEach('ensure it exists', () => { + cy.get('@ctx').then((ctx) => { + ensureMachineUserExists(ctx.api, machine.removeName); + cy.visit(machinesPath); + }); }); let loginName = machine.removeName; diff --git a/e2e/cypress/e2e/organization/organizations.cy.ts b/e2e/cypress/e2e/organization/organizations.cy.ts index f6ce9721a..83531c1ce 100644 --- a/e2e/cypress/e2e/organization/organizations.cy.ts +++ b/e2e/cypress/e2e/organization/organizations.cy.ts @@ -1,26 +1,25 @@ import { ensureOrgExists, ensureOrgIsDefault, isDefaultOrg } from 'support/api/orgs'; -import { apiAuth } from '../../support/api/apiauth'; import { v4 as uuidv4 } from 'uuid'; +import { Context } from 'support/commands'; const orgPath = `/org`; const orgNameOnCreation = 'e2eorgrename'; const testOrgNameChange = uuidv4(); +beforeEach(() => { + cy.context().as('ctx'); +}); + describe('organizations', () => { describe('rename', () => { beforeEach(() => { - apiAuth() - .as('api') - .then((api) => { - ensureOrgExists(api, orgNameOnCreation) - .as('newOrgId') - .then((newOrgId) => { - cy.visit(`${orgPath}?org=${newOrgId}`).as('orgsite'); - }); + cy.get('@ctx').then((ctx) => { + ensureOrgExists(ctx, orgNameOnCreation).then((newOrgId) => { + cy.visit(`${orgPath}?org=${newOrgId}`).as('orgsite'); }); + }); }); - it('should rename the organization', () => { cy.get('[data-e2e="actions"]').click(); cy.get('[data-e2e="rename"]', { timeout: 1000 }).should('be.visible').click(); @@ -31,24 +30,21 @@ describe('organizations', () => { cy.visit(orgPath); cy.get('[data-e2e="top-view-title"').should('contain', testOrgNameChange); }); - }); - const orgOverviewPath = `/orgs`; - const initialDefaultOrg = 'e2eorgolddefault'; - const orgNameForNewDefault = 'e2eorgnewdefault'; + const orgOverviewPath = `/orgs`; + const initialDefaultOrg = 'e2eorgolddefault'; + const orgNameForNewDefault = 'e2eorgnewdefault'; - describe('set default org', () => { - beforeEach(() => { - apiAuth() - .as('api') - .then((api) => { - ensureOrgExists(api, orgNameForNewDefault) + describe('set default org', () => { + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureOrgExists(ctx, orgNameForNewDefault) .as('newDefaultOrgId') .then(() => { - ensureOrgExists(api, initialDefaultOrg) + ensureOrgExists(ctx, initialDefaultOrg) .as('defaultOrg') .then((id) => { - ensureOrgIsDefault(api, id) + ensureOrgIsDefault(ctx, id) .as('orgWasDefault') .then(() => { cy.visit(`${orgOverviewPath}`).as('orgsite'); @@ -56,19 +52,24 @@ describe('organizations', () => { }); }); }); + }); + + it('should rename the organization', () => { + cy.get('@ctx').then((ctx) => { + const rowSelector = `tr:contains(${orgNameForNewDefault})`; + cy.get(rowSelector).find('[data-e2e="table-actions-button"]').click({ force: true }); + cy.get('[data-e2e="set-default-button"]', { timeout: 1000 }).should('be.visible').click(); + cy.shouldConfirmSuccess(); + cy.get('@newDefaultOrgId').then((newDefaultOrgId) => { + isDefaultOrg(ctx, newDefaultOrgId); + }); + }); + }); }); - it('should rename the organization', function () { - const rowSelector = `tr:contains(${orgNameForNewDefault})`; - cy.get(rowSelector).find('[data-e2e="table-actions-button"]').click({ force: true }); - cy.get('[data-e2e="set-default-button"]', { timeout: 1000 }).should('be.visible').click(); - cy.shouldConfirmSuccess(); - isDefaultOrg(this.api, this.newDefaultOrgId); + it('should add an organization with the personal account as org owner'); + describe('changing the current organization', () => { + it('should update displayed organization details'); }); }); - - it('should add an organization with the personal account as org owner'); - describe('changing the current organization', () => { - it('should update displayed organization details'); - }); }); diff --git a/e2e/cypress/e2e/permissions/permissions.cy.ts b/e2e/cypress/e2e/permissions/permissions.cy.ts index 7392514b4..d1755aa69 100644 --- a/e2e/cypress/e2e/permissions/permissions.cy.ts +++ b/e2e/cypress/e2e/permissions/permissions.cy.ts @@ -6,36 +6,46 @@ import { ensureHumanIsProjectMember, } from 'support/api/members'; import { ensureOrgExists } from 'support/api/orgs'; +import { ensureDomainPolicy } from 'support/api/policies'; import { ensureHumanUserExists, ensureUserDoesntExist } from 'support/api/users'; -import { loginname } from 'support/login/users'; -import { apiAuth } from '../../support/api/apiauth'; +import { Context } from 'support/commands'; import { ensureProjectExists, ensureProjectResourceDoesntExist, Roles } from '../../support/api/projects'; describe('permissions', () => { beforeEach(() => { - apiAuth().as('api'); + cy.context() + .as('ctx') + .then((ctx) => { + ensureDomainPolicy(ctx.api, false, true, false); + }); }); describe('management', () => { - const testManagerLoginname = loginname('e2ehumanmanager', Cypress.env('ORGANIZATION')); + const testManagerUsername = 'e2ehumanmanager'; function testAuthorizations( roles: string[], - beforeCreate: Mocha.HookFunction, - beforeMutate: Mocha.HookFunction, - navigate: Mocha.HookFunction, + beforeCreate: (ctx: Context) => void, + beforeMutate: (ctx: Context) => void, + navigate: () => void, ) { - beforeEach(function () { - ensureUserDoesntExist(this.api, testManagerLoginname); - ensureHumanUserExists(this.api, testManagerLoginname); + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureUserDoesntExist(ctx.api, testManagerUsername); + ensureHumanUserExists(ctx.api, testManagerUsername); + }); }); describe('create authorization', () => { - beforeEach(beforeCreate); - beforeEach(navigate); + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + beforeCreate(ctx); + navigate(); + }); + }); it('should add a manager', () => { cy.get('[data-e2e="add-member-button"]').click(); - cy.get('[data-e2e="add-member-input"]').type(testManagerLoginname); + cy.get('[data-e2e="add-member-input"]').type(testManagerUsername); cy.get('[data-e2e="user-option"]').first().click(); cy.contains('[data-e2e="role-checkbox"]', roles[0]).click(); cy.get('[data-e2e="confirm-add-member-button"]').click(); @@ -45,14 +55,15 @@ describe('permissions', () => { }); describe('mutate authorization', () => { - const rowSelector = `tr:contains(${testManagerLoginname})`; - - beforeEach(beforeMutate); - beforeEach(navigate); + const rowSelector = `tr:contains(${testManagerUsername})`; beforeEach(() => { - cy.contains('[data-e2e="member-avatar"]', 'ee').click(); - cy.get(rowSelector).as('managerRow'); + cy.get('@ctx').then((ctx) => { + beforeMutate(ctx); + navigate(); + cy.contains('[data-e2e="member-avatar"]', 'ee').click(); + cy.get(rowSelector).as('managerRow'); + }); }); it('should remove a manager', () => { @@ -88,14 +99,14 @@ describe('permissions', () => { testAuthorizations( roles.map((role) => role.display), - function () { - ensureHumanIsNotOrgMember(this.api, testManagerLoginname); + function (ctx: Context) { + ensureHumanIsNotOrgMember(ctx.api, testManagerUsername); }, - function () { - ensureHumanIsNotOrgMember(this.api, testManagerLoginname); + function (ctx: Context) { + ensureHumanIsNotOrgMember(ctx.api, testManagerUsername); ensureHumanIsOrgMember( - this.api, - testManagerLoginname, + ctx.api, + testManagerUsername, roles.map((role) => role.internal), ); }, @@ -108,12 +119,16 @@ describe('permissions', () => { describe('projects', () => { describe('owned projects', () => { - beforeEach(function () { - ensureProjectExists(this.api, 'e2eprojectpermission').as('projectId'); + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureProjectExists(ctx.api, 'e2eprojectpermission').as('projectId'); + }); }); - const visitOwnedProject: Mocha.HookFunction = function () { - cy.visit(`/projects/${this.projectId}`); + const visitOwnedProject = () => { + cy.get('@projectId').then((projectId) => { + cy.visit(`/projects/${projectId}`); + }); }; describe('authorizations', () => { @@ -124,17 +139,21 @@ describe('permissions', () => { testAuthorizations( roles.map((role) => role.display), - function () { - ensureHumanIsNotProjectMember(this.api, this.projectId, testManagerLoginname); + function (ctx) { + cy.get('@projectId').then((projectId) => { + ensureHumanIsNotProjectMember(ctx.api, projectId, testManagerUsername); + }); }, - function () { - ensureHumanIsNotProjectMember(this.api, this.projectId, testManagerLoginname); - ensureHumanIsProjectMember( - this.api, - this.projectId, - testManagerLoginname, - roles.map((role) => role.internal), - ); + function (ctx) { + cy.get('@projectId').then((projectId) => { + ensureHumanIsNotProjectMember(ctx.api, projectId, testManagerUsername); + ensureHumanIsProjectMember( + ctx.api, + projectId, + testManagerUsername, + roles.map((role) => role.internal), + ); + }); }, visitOwnedProject, ); @@ -143,12 +162,15 @@ describe('permissions', () => { describe('roles', () => { const testRoleName = 'e2eroleundertestname'; - beforeEach(function () { - ensureProjectResourceDoesntExist(this.api, this.projectId, Roles, testRoleName); + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + cy.get('@projectId').then((projectId) => { + ensureProjectResourceDoesntExist(ctx.api, projectId, Roles, testRoleName); + visitOwnedProject(); + }); + }); }); - beforeEach(visitOwnedProject); - it('should add a role', () => { cy.get('[data-e2e="sidenav-element-roles"]').click(); cy.get('[data-e2e="add-new-role"]').click(); @@ -164,21 +186,25 @@ describe('permissions', () => { }); describe('granted projects', () => { - beforeEach(function () { - ensureOrgExists(this.api, 'e2eforeignorg') - .as('foreignOrgId') - .then((foreignOrgId) => { - ensureProjectExists(this.api, 'e2eprojectgrants', foreignOrgId) - .as('projectId') - .then((projectId) => { - ensureProjectGrantExists(this.api, foreignOrgId, projectId).as('grantId'); + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureOrgExists(ctx, 'e2eforeignorg').then((foreignOrgId) => { + ensureProjectExists(ctx.api, 'e2eprojectgrants', foreignOrgId) + .as('foreignProjectId') + .then((foreignProjectId) => { + ensureProjectGrantExists(ctx, foreignOrgId, foreignProjectId).as('grantId'); }); }); + }); }); - const visitGrantedProject: Mocha.HookFunction = function () { - cy.visit(`/granted-projects/${this.projectId}/grant/${this.grantId}`); - }; + function visitGrantedProject() { + cy.get('@foreignProjectId').then((foreignProjectId) => { + cy.get('@grantId').then((grantId) => { + cy.visit(`/granted-projects/${foreignProjectId}/grant/${grantId}`); + }); + }); + } describe('authorizations', () => { const roles = [ @@ -188,18 +214,26 @@ describe('permissions', () => { testAuthorizations( roles.map((role) => role.display), - function () { - ensureHumanIsNotProjectMember(this.api, this.projectId, testManagerLoginname, this.grantId); + function (ctx: Context) { + cy.get('@foreignProjectId').then((foreignProjectId) => { + cy.get('@grantId').then((grantId) => { + ensureHumanIsNotProjectMember(ctx.api, foreignProjectId, testManagerUsername, grantId); + }); + }); }, - function () { - ensureHumanIsNotProjectMember(this.api, this.projectId, testManagerLoginname, this.grantId); - ensureHumanIsProjectMember( - this.api, - this.projectId, - testManagerLoginname, - roles.map((role) => role.internal), - this.grantId, - ); + function (ctx: Context) { + cy.get('@foreignProjectId').then((foreignProjectId) => { + cy.get('@grantId').then((grantId) => { + ensureHumanIsNotProjectMember(ctx.api, foreignProjectId, testManagerUsername, grantId); + ensureHumanIsProjectMember( + ctx.api, + foreignProjectId, + testManagerUsername, + roles.map((role) => role.internal), + grantId, + ); + }); + }); }, visitGrantedProject, ); @@ -207,42 +241,42 @@ describe('permissions', () => { }); }); }); +}); - describe('validations', () => { - describe('owned projects', () => { - describe('no ownership', () => { - it('a user without project global ownership can ...'); - it('a user without project global ownership can not ...'); - }); - describe('project owner viewer global', () => { - it('a project owner viewer global additionally can ...'); - it('a project owner viewer global still can not ...'); - }); - describe('project owner global', () => { - it('a project owner global additionally can ...'); - it('a project owner global still can not ...'); - }); +describe('validations', () => { + describe('owned projects', () => { + describe('no ownership', () => { + it('a user without project global ownership can ...'); + it('a user without project global ownership can not ...'); }); + describe('project owner viewer global', () => { + it('a project owner viewer global additionally can ...'); + it('a project owner viewer global still can not ...'); + }); + describe('project owner global', () => { + it('a project owner global additionally can ...'); + it('a project owner global still can not ...'); + }); + }); - describe('granted projects', () => { - describe('no ownership', () => { - it('a user without project grant ownership can ...'); - it('a user without project grant ownership can not ...'); - }); - describe('project grant owner viewer', () => { - it('a project grant owner viewer additionally can ...'); - it('a project grant owner viewer still can not ...'); - }); - describe('project grant owner', () => { - it('a project grant owner additionally can ...'); - it('a project grant owner still can not ...'); - }); + describe('granted projects', () => { + describe('no ownership', () => { + it('a user without project grant ownership can ...'); + it('a user without project grant ownership can not ...'); }); - describe('organization', () => { - describe('org owner', () => { - it('a project owner global can ...'); - it('a project owner global can not ...'); - }); + describe('project grant owner viewer', () => { + it('a project grant owner viewer additionally can ...'); + it('a project grant owner viewer still can not ...'); + }); + describe('project grant owner', () => { + it('a project grant owner additionally can ...'); + it('a project grant owner still can not ...'); + }); + }); + describe('organization', () => { + describe('org owner', () => { + it('a project owner global can ...'); + it('a project owner global can not ...'); }); }); }); diff --git a/e2e/cypress/e2e/projects/projects.cy.ts b/e2e/cypress/e2e/projects/projects.cy.ts index 0ff46f31c..36768a153 100644 --- a/e2e/cypress/e2e/projects/projects.cy.ts +++ b/e2e/cypress/e2e/projects/projects.cy.ts @@ -1,18 +1,20 @@ -import { apiAuth } from '../../support/api/apiauth'; +import { Context } from 'support/commands'; import { ensureProjectDoesntExist, ensureProjectExists } from '../../support/api/projects'; describe('projects', () => { beforeEach(() => { - apiAuth().as('api'); + cy.context().as('ctx'); }); const testProjectNameCreate = 'e2eprojectcreate'; const testProjectNameDelete = 'e2eprojectdelete'; describe('add project', () => { - beforeEach(`ensure it doesn't exist already`, function () { - ensureProjectDoesntExist(this.api, testProjectNameCreate); - cy.visit(`/projects`); + beforeEach(`ensure it doesn't exist already`, () => { + cy.get('@ctx').then((ctx) => { + ensureProjectDoesntExist(ctx.api, testProjectNameCreate); + cy.visit(`/projects`); + }); }); it('should add a project', () => { @@ -26,9 +28,11 @@ describe('projects', () => { }); describe('edit project', () => { - beforeEach('ensure it exists', function () { - ensureProjectExists(this.api, testProjectNameDelete); - cy.visit(`/projects`); + beforeEach('ensure it exists', () => { + cy.get('@ctx').then((ctx) => { + ensureProjectExists(ctx.api, testProjectNameDelete); + cy.visit(`/projects`); + }); }); describe('remove project', () => { diff --git a/e2e/cypress/e2e/quotas/quotas.cy.ts b/e2e/cypress/e2e/quotas/quotas.cy.ts new file mode 100644 index 000000000..408752f82 --- /dev/null +++ b/e2e/cypress/e2e/quotas/quotas.cy.ts @@ -0,0 +1,248 @@ +import { addQuota, ensureQuotaIsAdded, ensureQuotaIsRemoved, removeQuota, Unit } from 'support/api/quota'; +import { createHumanUser, ensureUserDoesntExist } from 'support/api/users'; +import { Context } from 'support/commands'; +import { ZITADELWebhookEvent } from 'support/types'; + +beforeEach(() => { + cy.context().as('ctx'); +}); + +describe('quotas', () => { + describe('management', () => { + describe('add one quota', () => { + it('should add a quota only once per unit', () => { + cy.get('@ctx').then((ctx) => { + addQuota(ctx, Unit.AuthenticatedRequests, true, 1); + addQuota(ctx, Unit.AuthenticatedRequests, true, 1, undefined, undefined, undefined, false).then((res) => { + expect(res.status).to.equal(409); + }); + }); + }); + + describe('add two quotas', () => { + it('should add a quota for each unit', () => { + cy.get('@ctx').then((ctx) => { + addQuota(ctx, Unit.AuthenticatedRequests, true, 1); + addQuota(ctx, Unit.ExecutionSeconds, true, 1); + }); + }); + }); + }); + + describe('edit', () => { + describe('remove one quota', () => { + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureQuotaIsAdded(ctx, Unit.AuthenticatedRequests, true, 1); + }); + }); + it('should remove a quota only once per unit', () => { + cy.get('@ctx').then((ctx) => { + removeQuota(ctx, Unit.AuthenticatedRequests); + }); + cy.get('@ctx').then((ctx) => { + removeQuota(ctx, Unit.AuthenticatedRequests, false).then((res) => { + expect(res.status).to.equal(404); + }); + }); + }); + + describe('remove two quotas', () => { + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureQuotaIsAdded(ctx, Unit.AuthenticatedRequests, true, 1); + ensureQuotaIsAdded(ctx, Unit.ExecutionSeconds, true, 1); + }); + }); + it('should remove a quota for each unit', () => { + cy.get('@ctx').then((ctx) => { + removeQuota(ctx, Unit.AuthenticatedRequests); + removeQuota(ctx, Unit.ExecutionSeconds); + }); + }); + }); + }); + }); + }); + + describe('usage', () => { + beforeEach(() => { + cy.get('@ctx') + .then((ctx) => { + return [ + `${ctx.api.oidcBaseURL}/userinfo`, + `${ctx.api.authBaseURL}/users/me`, + `${ctx.api.mgmtBaseURL}/iam`, + `${ctx.api.adminBaseURL}/instances/me`, + `${ctx.api.oauthBaseURL}/keys`, + `${ctx.api.samlBaseURL}/certificate`, + ]; + }) + .as('authenticatedUrls'); + }); + + describe('authenticated requests', () => { + const testUserName = 'shouldNotBeCreated'; + beforeEach(() => { + cy.get>('@authenticatedUrls').then((urls) => { + cy.get('@ctx').then((ctx) => { + ensureUserDoesntExist(ctx.api, testUserName); + ensureQuotaIsAdded(ctx, Unit.AuthenticatedRequests, true, urls.length); + cy.task('runSQL', `TRUNCATE logstore.access;`); + }); + }); + }); + + it('authenticated requests are limited', () => { + cy.get>('@authenticatedUrls').then((urls) => { + cy.get('@ctx').then((ctx) => { + const start = new Date(); + urls.forEach((url) => { + cy.request({ + url: url, + method: 'GET', + auth: { + bearer: ctx.api.token, + }, + }); + }); + const expiresMax = new Date(); + expiresMax.setMinutes(expiresMax.getMinutes() + 2); + cy.getCookie('zitadel.quota.limiting').then((cookie) => { + expect(cookie.value).to.equal('false'); + const cookieExpiry = new Date(); + cookieExpiry.setTime(cookie.expiry * 1000); + expect(cookieExpiry).to.be.within(start, expiresMax); + }); + cy.request({ + url: urls[0], + method: 'GET', + auth: { + bearer: ctx.api.token, + }, + failOnStatusCode: false, + }).then((res) => { + expect(res.status).to.equal(429); + }); + cy.getCookie('zitadel.quota.limiting').then((cookie) => { + expect(cookie.value).to.equal('true'); + }); + createHumanUser(ctx.api, testUserName, false).then((res) => { + expect(res.status).to.equal(429); + }); + ensureQuotaIsRemoved(ctx, Unit.AuthenticatedRequests); + createHumanUser(ctx.api, testUserName); + }); + }); + }); + }); + + describe('notifications', () => { + const callURL = `http://${Cypress.env('WEBHOOK_HANDLER_HOST')}:${Cypress.env('WEBHOOK_HANDLER_PORT')}/do_something`; + + beforeEach(() => cy.task('resetWebhookEvents')); + + const amount = 100; + const percent = 10; + const usage = 25; + + describe('without repetition', () => { + beforeEach(() => { + cy.get('@ctx').then((ctx) => { + ensureQuotaIsAdded(ctx, Unit.AuthenticatedRequests, false, amount, [ + { + callUrl: callURL, + percent: percent, + repeat: false, + }, + ]); + cy.task('runSQL', `TRUNCATE logstore.access;`); + }); + }); + + it('fires once with the expected payload', () => { + cy.get>('@authenticatedUrls').then((urls) => { + cy.get('@ctx').then((ctx) => { + for (let i = 0; i < usage; i++) { + cy.request({ + url: urls[0], + method: 'GET', + auth: { + bearer: ctx.api.token, + }, + }); + } + }); + cy.waitUntil(() => + cy.task>('handledWebhookEvents').then((events) => { + if (events.length != 1) { + return false; + } + return Cypress._.matches({ + callURL: callURL, + threshold: percent, + unit: 1, + usage: percent, + })(events[0]); + }), + ); + }); + }); + }); + + describe('with repetition', () => { + beforeEach(() => { + cy.get>('@authenticatedUrls').then((urls) => { + cy.get('@ctx').then((ctx) => { + ensureQuotaIsAdded(ctx, Unit.AuthenticatedRequests, false, amount, [ + { + callUrl: callURL, + percent: percent, + repeat: true, + }, + ]); + cy.task('runSQL', `TRUNCATE logstore.access;`); + }); + }); + }); + + it('fires repeatedly with the expected payloads', () => { + cy.get>('@authenticatedUrls').then((urls) => { + cy.get('@ctx').then((ctx) => { + for (let i = 0; i < usage; i++) { + cy.request({ + url: urls[0], + method: 'GET', + auth: { + bearer: ctx.api.token, + }, + }); + } + }); + }); + cy.waitUntil(() => + cy.task>('handledWebhookEvents').then((events) => { + if (events.length != 1) { + return false; + } + for (let i = 0; i < events.length; i++) { + const threshold = percent * (i + 1); + if ( + !Cypress._.matches({ + callURL: callURL, + threshold: threshold, + unit: 1, + usage: threshold, + })(events[i]) + ) { + return false; + } + } + return true; + }), + ); + }); + }); + }); + }); +}); diff --git a/e2e/cypress/e2e/register/register.cy.ts b/e2e/cypress/e2e/register/register.cy.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/e2e/cypress/e2e/settings/private-labeling.cy.ts b/e2e/cypress/e2e/settings/private-labeling.cy.ts index ac8a1d313..ea9bc16c9 100644 --- a/e2e/cypress/e2e/settings/private-labeling.cy.ts +++ b/e2e/cypress/e2e/settings/private-labeling.cy.ts @@ -1,4 +1,5 @@ -import { apiAuth, API } from '../../support/api/apiauth'; +import { Context } from 'mocha'; +import { apiAuth } from '../../support/api/apiauth'; import { Policy, resetPolicy } from '../../support/api/policies'; import { login, User } from '../../support/login/users'; @@ -7,8 +8,6 @@ describe('private labeling', () => { [User.OrgOwner].forEach((user) => { describe(`as user "${user}"`, () => { - let api: API; - beforeEach(() => { login(user); cy.visit(orgPath); @@ -37,34 +36,27 @@ function customize(theme: string, user: User) { }); it('should update a logo', () => { - cy.get('[data-e2e="image-part-logo"]') - .find('input') - .then(function (el) { - const blob = Cypress.Blob.base64StringToBlob(this.logo, 'image/png'); - const file = new File([blob], 'images/logo.png', { type: 'image/png' }); - const list = new DataTransfer(); + cy.get('@ctx').then((ctx) => { + cy.get('[data-e2e="image-part-logo"]') + .find('input') + .then(function (el) { + const blob = Cypress.Blob.base64StringToBlob(ctx.logo, 'image/png'); + const file = new File([blob], 'images/logo.png', { type: 'image/png' }); + const list = new DataTransfer(); - list.items.add(file); - const myFileList = list.files; + list.items.add(file); + const myFileList = list.files; - el[0].files = myFileList; - el[0].dispatchEvent(new Event('change', { bubbles: true })); - }); + el[0].files = myFileList; + el[0].dispatchEvent(new Event('change', { bubbles: true })); + }); + }); }); it('should delete a logo'); }); it('should update an icon'); it('should delete an icon'); - it.skip('should update the background color', () => { - cy.contains('[data-e2e="color"]', 'Background Color').find('button').click(); // TODO: select data-e2e - cy.get('color-editable-input').find('input').clear().type('#ae44dc'); - cy.get('[data-e2e="save-colors-button"]').click(); - cy.get('[data-e2e="header-user-avatar"]').click(); - cy.contains('Logout All Users').click(); // TODO: select data-e2e - login(User.LoginPolicyUser, undefined, true, null, () => { - cy.pause(); - }); - }); + it('should update the background color'); it('should update the primary color'); it('should update the warning color'); it('should update the font color'); diff --git a/e2e/cypress/support/api/apiauth.ts b/e2e/cypress/support/api/apiauth.ts index b351aedd3..fd1720e3e 100644 --- a/e2e/cypress/support/api/apiauth.ts +++ b/e2e/cypress/support/api/apiauth.ts @@ -1,21 +1,36 @@ import { login, User } from 'support/login/users'; -import { API } from './types'; +import { API, SystemAPI, Token } from './types'; const authHeaderKey = 'Authorization', - orgIdHeaderKey = 'x-zitadel-orgid'; + orgIdHeaderKey = 'x-zitadel-orgid', + backendUrl = Cypress.env('BACKEND_URL'); export function apiAuth(): Cypress.Chainable { return login(User.IAMAdminUser, 'Password1!', false, true).then((token) => { return { token: token, - mgmtBaseURL: `${Cypress.env('BACKEND_URL')}/management/v1`, - adminBaseURL: `${Cypress.env('BACKEND_URL')}/admin/v1`, + mgmtBaseURL: `${backendUrl}/management/v1`, + adminBaseURL: `${backendUrl}/admin/v1`, + authBaseURL: `${backendUrl}/auth/v1`, + assetsBaseURL: `${backendUrl}/assets/v1`, + oauthBaseURL: `${backendUrl}/oauth/v2`, + oidcBaseURL: `${backendUrl}/oidc/v1`, + samlBaseURL: `${backendUrl}/saml/v2`, }; }); } -export function requestHeaders(api: API, orgId?: number): object { - const headers = { [authHeaderKey]: `Bearer ${api.token}` }; +export function systemAuth(): Cypress.Chainable { + return cy.task('systemToken').then((token) => { + return { + token: token, + baseURL: `${backendUrl}/system/v1`, + }; + }); +} + +export function requestHeaders(token: Token, orgId?: string): object { + const headers = { [authHeaderKey]: `Bearer ${token.token}` }; if (orgId) { headers[orgIdHeaderKey] = orgId; } diff --git a/e2e/cypress/support/api/ensure.ts b/e2e/cypress/support/api/ensure.ts index 7b0faebd9..cca7322ad 100644 --- a/e2e/cypress/support/api/ensure.ts +++ b/e2e/cypress/support/api/ensure.ts @@ -1,20 +1,20 @@ import { requestHeaders } from './apiauth'; import { findFromList as mapFromList, searchSomething } from './search'; -import { API, Entity, SearchResult } from './types'; +import { API, Entity, SearchResult, Token } from './types'; export function ensureItemExists( - api: API, + token: Token, searchPath: string, findInList: (entity: Entity) => boolean, createPath: string, body: Entity, - orgId?: number, + orgId?: string, newItemIdField: string = 'id', searchItemIdField?: string, -): Cypress.Chainable { +) { return ensureSomething( - api, - () => searchSomething(api, searchPath, 'POST', mapFromList(findInList, searchItemIdField), orgId), + token, + () => searchSomething(token, searchPath, 'POST', mapFromList(findInList, searchItemIdField), orgId), () => createPath, 'POST', body, @@ -25,15 +25,15 @@ export function ensureItemExists( } export function ensureItemDoesntExist( - api: API, + token: Token, searchPath: string, findInList: (entity: Entity) => boolean, deletePath: (entity: Entity) => string, - orgId?: number, + orgId?: string, ): Cypress.Chainable { return ensureSomething( - api, - () => searchSomething(api, searchPath, 'POST', mapFromList(findInList), orgId), + token, + () => searchSomething(token, searchPath, 'POST', mapFromList(findInList), orgId), deletePath, 'DELETE', null, @@ -47,8 +47,8 @@ export function ensureSetting( mapResult: (entity: any) => SearchResult, createPath: string, body: any, - orgId?: number, -): Cypress.Chainable { + orgId?: string, +): Cypress.Chainable { return ensureSomething( api, () => searchSomething(api, path, 'GET', mapResult, orgId), @@ -79,38 +79,38 @@ function awaitDesired( } interface EnsuredResult { - id: number; + id: string; sequence: number; } export function ensureSomething( - api: API, + token: Token, search: () => Cypress.Chainable, apiPath: (entity: Entity) => string, ensureMethod: string, body: Entity, expectEntity: (entity: Entity) => boolean, - mapId?: (body: any) => number, - orgId?: number, -): Cypress.Chainable { + mapId?: (body: any) => string, + orgId?: string, +): Cypress.Chainable { return search() - .then((sRes) => { + .then((sRes) => { if (expectEntity(sRes.entity)) { - return cy.wrap({ id: sRes.id, sequence: sRes.sequence }); + return cy.wrap({ id: sRes.id, sequence: sRes.sequence }); } return cy .request({ method: ensureMethod, url: apiPath(sRes.entity), - headers: requestHeaders(api, orgId), + headers: requestHeaders(token, orgId), body: body, failOnStatusCode: false, followRedirect: false, }) .then((cRes) => { expect(cRes.status).to.equal(200); - return { + return { id: mapId ? mapId(cRes.body) : undefined, sequence: sRes.sequence, }; @@ -118,7 +118,7 @@ export function ensureSomething( }) .then((data) => { return awaitDesired(90, expectEntity, search, data.sequence).then(() => { - return cy.wrap(data.id); + return cy.wrap(data.id); }); }); } diff --git a/e2e/cypress/support/api/grants.ts b/e2e/cypress/support/api/grants.ts index a178768ee..5f6f5f476 100644 --- a/e2e/cypress/support/api/grants.ts +++ b/e2e/cypress/support/api/grants.ts @@ -1,18 +1,14 @@ +import { Context } from 'support/commands'; import { ensureItemExists } from './ensure'; import { getOrgUnderTest } from './orgs'; -import { API } from './types'; -export function ensureProjectGrantExists( - api: API, - foreignOrgId: number, - foreignProjectId: number, -): Cypress.Chainable { - return getOrgUnderTest(api).then((orgUnderTest) => { +export function ensureProjectGrantExists(ctx: Context, foreignOrgId: string, foreignProjectId: string) { + return getOrgUnderTest(ctx).then((orgUnderTest) => { return ensureItemExists( - api, - `${api.mgmtBaseURL}/projectgrants/_search`, + ctx.api, + `${ctx.api.mgmtBaseURL}/projectgrants/_search`, (grant: any) => grant.grantedOrgId == orgUnderTest && grant.projectId == foreignProjectId, - `${api.mgmtBaseURL}/projects/${foreignProjectId}/grants`, + `${ctx.api.mgmtBaseURL}/projects/${foreignProjectId}/grants`, { granted_org_id: orgUnderTest }, foreignOrgId, 'grantId', diff --git a/e2e/cypress/support/api/instances.ts b/e2e/cypress/support/api/instances.ts new file mode 100644 index 000000000..ac6a7a839 --- /dev/null +++ b/e2e/cypress/support/api/instances.ts @@ -0,0 +1,31 @@ +import { SystemAPI } from './types'; + +export function instanceUnderTest(api: SystemAPI): Cypress.Chainable { + return cy + .request({ + method: 'POST', + url: `${api.baseURL}/instances/_search`, + auth: { + bearer: api.token, + }, + }) + .then((res) => { + const instances = >res.body.result; + expect(instances.length).to.equal( + 1, + 'instanceUnderTest just supports running against an API with exactly one instance, yet', + ); + return instances[0].id; + }); +} + +export function getInstance(api: SystemAPI, instanceId: string, failOnStatusCode = true) { + return cy.request({ + method: 'GET', + url: `${api.baseURL}/instances/${instanceId}`, + auth: { + bearer: api.token, + }, + failOnStatusCode: failOnStatusCode, + }); +} diff --git a/e2e/cypress/support/api/members.ts b/e2e/cypress/support/api/members.ts index ce718663f..bcc7ce72c 100644 --- a/e2e/cypress/support/api/members.ts +++ b/e2e/cypress/support/api/members.ts @@ -2,7 +2,7 @@ import { ensureItemDoesntExist, ensureItemExists } from './ensure'; import { findFromList, searchSomething } from './search'; import { API } from './types'; -export function ensureHumanIsNotOrgMember(api: API, username: string): Cypress.Chainable { +export function ensureHumanIsNotOrgMember(api: API, username: string) { return ensureItemDoesntExist( api, `${api.mgmtBaseURL}/orgs/me/members/_search`, @@ -11,7 +11,7 @@ export function ensureHumanIsNotOrgMember(api: API, username: string): Cypress.C ); } -export function ensureHumanIsOrgMember(api: API, username: string, roles: string[]): Cypress.Chainable { +export function ensureHumanIsOrgMember(api: API, username: string, roles: string[]) { return searchSomething( api, `${api.mgmtBaseURL}/users/_search`, @@ -37,13 +37,13 @@ export function ensureHumanIsNotProjectMember( api: API, projectId: string, username: string, - grantId?: number, -): Cypress.Chainable { + grantId?: string, +): Cypress.Chainable { return ensureItemDoesntExist( api, `${api.mgmtBaseURL}/projects/${projectId}/${grantId ? `grants/${grantId}/` : ''}members/_search`, (member: any) => (member.preferredLoginName).startsWith(username), - (member) => `${api.mgmtBaseURL}/projects/${projectId}${grantId ? `grants/${grantId}/` : ''}/members/${member.userId}`, + (member) => `${api.mgmtBaseURL}/projects/${projectId}/${grantId ? `grants/${grantId}/` : ''}members/${member.userId}`, ); } @@ -52,8 +52,8 @@ export function ensureHumanIsProjectMember( projectId: string, username: string, roles: string[], - grantId?: number, -): Cypress.Chainable { + grantId?: string, +): Cypress.Chainable { return searchSomething( api, `${api.mgmtBaseURL}/users/_search`, diff --git a/e2e/cypress/support/api/oidc-settings.ts b/e2e/cypress/support/api/oidc-settings.ts index dbc676a23..652dd25fe 100644 --- a/e2e/cypress/support/api/oidc-settings.ts +++ b/e2e/cypress/support/api/oidc-settings.ts @@ -7,7 +7,7 @@ export function ensureOIDCSettingsSet( idTokenLifetime: number, refreshTokenExpiration: number, refreshTokenIdleExpiration: number, -): Cypress.Chainable { +) { return ensureSetting( api, `${api.adminBaseURL}/settings/oidc`, diff --git a/e2e/cypress/support/api/orgs.ts b/e2e/cypress/support/api/orgs.ts index ceaa98b5c..6af76560a 100644 --- a/e2e/cypress/support/api/orgs.ts +++ b/e2e/cypress/support/api/orgs.ts @@ -3,20 +3,21 @@ import { searchSomething } from './search'; import { API } from './types'; import { host } from '../login/users'; import { requestHeaders } from './apiauth'; +import { Context } from 'support/commands'; -export function ensureOrgExists(api: API, name: string): Cypress.Chainable { +export function ensureOrgExists(ctx: Context, name: string) { return ensureSomething( - api, + ctx.api, () => searchSomething( - api, - encodeURI(`${api.mgmtBaseURL}/global/orgs/_by_domain?domain=${name}.${host(Cypress.config('baseUrl'))}`), + ctx.api, + encodeURI(`${ctx.api.mgmtBaseURL}/global/orgs/_by_domain?domain=${name}.${host(Cypress.config('baseUrl'))}`), 'GET', (res) => { return { entity: res.org, id: res.org?.id, sequence: parseInt(res.org?.details?.sequence) }; }, ), - () => `${api.mgmtBaseURL}/orgs`, + () => `${ctx.api.mgmtBaseURL}/orgs`, 'POST', { name: name }, (org) => org?.name === name, @@ -24,13 +25,13 @@ export function ensureOrgExists(api: API, name: string): Cypress.Chainable { +export function isDefaultOrg(ctx: Context, orgId: string): Cypress.Chainable { console.log('huhu', orgId); return cy .request({ method: 'GET', - url: encodeURI(`${api.mgmtBaseURL}/iam`), - headers: requestHeaders(api, orgId), + url: encodeURI(`${ctx.api.mgmtBaseURL}/iam`), + headers: requestHeaders(ctx.api, orgId), }) .then((res) => { const { defaultOrgId } = res.body; @@ -39,12 +40,12 @@ export function isDefaultOrg(api: API, orgId: number): Cypress.Chainable { +export function ensureOrgIsDefault(ctx: Context, orgId: string): Cypress.Chainable { return cy .request({ method: 'GET', - url: encodeURI(`${api.mgmtBaseURL}/iam`), - headers: requestHeaders(api, orgId), + url: encodeURI(`${ctx.api.mgmtBaseURL}/iam`), + headers: requestHeaders(ctx.api, orgId), }) .then((res) => { return res.body; @@ -56,8 +57,8 @@ export function ensureOrgIsDefault(api: API, orgId: number): Cypress.Chainable { - return searchSomething(api, `${api.mgmtBaseURL}/orgs/me`, 'GET', (res) => { +export function getOrgUnderTest(ctx: Context): Cypress.Chainable { + return searchSomething(ctx.api, `${ctx.api.mgmtBaseURL}/orgs/me`, 'GET', (res) => { return { entity: res.org, id: res.org.id, sequence: parseInt(res.org.details.sequence) }; }).then((res) => res.entity.id); } diff --git a/e2e/cypress/support/api/policies.ts b/e2e/cypress/support/api/policies.ts index 3ad70b035..bc86b6107 100644 --- a/e2e/cypress/support/api/policies.ts +++ b/e2e/cypress/support/api/policies.ts @@ -22,7 +22,7 @@ export function ensureDomainPolicy( userLoginMustBeDomain: boolean, validateOrgDomains: boolean, smtpSenderAddressMatchesInstanceDomain: boolean, -): Cypress.Chainable { +) { return ensureSetting( api, `${api.adminBaseURL}/policies/domain`, diff --git a/e2e/cypress/support/api/projects.ts b/e2e/cypress/support/api/projects.ts index df0d9179e..ac368c775 100644 --- a/e2e/cypress/support/api/projects.ts +++ b/e2e/cypress/support/api/projects.ts @@ -1,7 +1,7 @@ import { ensureItemDoesntExist, ensureItemExists } from './ensure'; -import { API } from './types'; +import { API, Entity } from './types'; -export function ensureProjectExists(api: API, projectName: string, orgId?: number): Cypress.Chainable { +export function ensureProjectExists(api: API, projectName: string, orgId?: string) { return ensureItemExists( api, `${api.mgmtBaseURL}/projects/_search`, @@ -12,7 +12,7 @@ export function ensureProjectExists(api: API, projectName: string, orgId?: numbe ); } -export function ensureProjectDoesntExist(api: API, projectName: string, orgId?: number): Cypress.Chainable { +export function ensureProjectDoesntExist(api: API, projectName: string, orgId?: string) { return ensureItemDoesntExist( api, `${api.mgmtBaseURL}/projects/_search`, @@ -32,22 +32,22 @@ export const Roles = new ResourceType('roles', 'key', 'key'); export function ensureProjectResourceDoesntExist( api: API, - projectId: number, + projectId: string, resourceType: ResourceType, resourceName: string, - orgId?: number, + orgId?: string, ): Cypress.Chainable { return ensureItemDoesntExist( api, `${api.mgmtBaseURL}/projects/${projectId}/${resourceType.resourcePath}/_search`, - (resource: any) => resource[resourceType.compareProperty] === resourceName, - (resource) => + (resource: Entity) => resource[resourceType.compareProperty] === resourceName, + (resource: Entity) => `${api.mgmtBaseURL}/projects/${projectId}/${resourceType.resourcePath}/${resource[resourceType.identifierProperty]}`, orgId, ); } -export function ensureApplicationExists(api: API, projectId: number, appName: string): Cypress.Chainable { +export function ensureApplicationExists(api: API, projectId: number, appName: string) { return ensureItemExists( api, `${api.mgmtBaseURL}/projects/${projectId}/${Apps.resourcePath}/_search`, diff --git a/e2e/cypress/support/api/quota.ts b/e2e/cypress/support/api/quota.ts new file mode 100644 index 000000000..884fcb2b6 --- /dev/null +++ b/e2e/cypress/support/api/quota.ts @@ -0,0 +1,84 @@ +import { Context } from 'support/commands'; + +export enum Unit { + Unimplemented, + AuthenticatedRequests, + ExecutionSeconds, +} + +interface notification { + percent: number; + repeat?: boolean; + callUrl: string; +} + +export function addQuota( + ctx: Context, + unit: Unit = Unit.AuthenticatedRequests, + limit: boolean, + amount: number, + notifications?: Array, + from: Date = (() => { + const date = new Date(); + date.setMonth(0, 1); + date.setMinutes(0, 0, 0); + // default to start of current year + return date; + })(), + intervalSeconds: string = `${315_576_000_000}s`, // proto max duration is 1000 years + failOnStatusCode = true, +): Cypress.Chainable> { + return cy.request({ + method: 'POST', + url: `${ctx.system.baseURL}/instances/${ctx.instanceId}/quotas`, + auth: { + bearer: ctx.system.token, + }, + body: { + unit: unit, + amount: amount, + resetInterval: intervalSeconds, + limit: limit, + from: from, + notifications: notifications, + }, + failOnStatusCode: failOnStatusCode, + }); +} + +export function ensureQuotaIsAdded( + ctx: Context, + unit: Unit, + limit: boolean, + amount?: number, + notifications?: Array, + from?: Date, + intervalSeconds?: string, +): Cypress.Chainable { + return addQuota(ctx, unit, limit, amount, notifications, from, intervalSeconds, false).then((res) => { + if (!res.isOkStatusCode) { + expect(res.status).to.equal(409); + } + return null; + }); +} + +export function removeQuota(ctx: Context, unit: Unit, failOnStatusCode = true): Cypress.Chainable> { + return cy.request({ + method: 'DELETE', + url: `${ctx.system.baseURL}/instances/${ctx.instanceId}/quotas/${unit}`, + auth: { + bearer: ctx.system.token, + }, + failOnStatusCode: failOnStatusCode, + }); +} + +export function ensureQuotaIsRemoved(ctx: Context, unit?: Unit): Cypress.Chainable { + return removeQuota(ctx, unit, false).then((res) => { + if (!res.isOkStatusCode) { + expect(res.status).to.equal(404); + } + return null; + }); +} diff --git a/e2e/cypress/support/api/search.ts b/e2e/cypress/support/api/search.ts index a2077b1de..a0ed5895a 100644 --- a/e2e/cypress/support/api/search.ts +++ b/e2e/cypress/support/api/search.ts @@ -1,18 +1,18 @@ import { requestHeaders } from './apiauth'; -import { API, Entity, SearchResult } from './types'; +import { API, Entity, SearchResult, Token } from './types'; export function searchSomething( - api: API, + token: Token, searchPath: string, method: string, mapResult: (body: any) => SearchResult, - orgId?: number, + orgId?: string, ): Cypress.Chainable { return cy .request({ method: method, url: searchPath, - headers: requestHeaders(api, orgId), + headers: requestHeaders(token, orgId), failOnStatusCode: method == 'POST', }) .then((res) => { diff --git a/e2e/cypress/support/api/types.ts b/e2e/cypress/support/api/types.ts index 58af9cebd..39a8094b8 100644 --- a/e2e/cypress/support/api/types.ts +++ b/e2e/cypress/support/api/types.ts @@ -1,13 +1,25 @@ -export interface API { +export interface Token { token: string; +} + +export interface API extends Token { mgmtBaseURL: string; adminBaseURL: string; + authBaseURL: string; + assetsBaseURL: string; + oidcBaseURL: string; + oauthBaseURL: string; + samlBaseURL: string; +} + +export interface SystemAPI extends Token { + baseURL: string; } export type SearchResult = { entity: Entity | null; sequence: number; - id: number; + id: string; }; // Entity is an object but not a function diff --git a/e2e/cypress/support/api/users.ts b/e2e/cypress/support/api/users.ts index 5f650bc22..7d2ccae83 100644 --- a/e2e/cypress/support/api/users.ts +++ b/e2e/cypress/support/api/users.ts @@ -1,31 +1,23 @@ +import { requestHeaders } from './apiauth'; import { ensureItemDoesntExist, ensureItemExists } from './ensure'; import { API } from './types'; -export function ensureHumanUserExists(api: API, username: string): Cypress.Chainable { +export function ensureHumanUserExists(api: API, username: string) { return ensureItemExists( api, `${api.mgmtBaseURL}/users/_search`, (user: any) => user.userName === username, `${api.mgmtBaseURL}/users/human`, { + ...defaultHuman, user_name: username, - profile: { - first_name: 'e2efirstName', - last_name: 'e2elastName', - }, - email: { - email: 'e2e@email.ch', - }, - phone: { - phone: '+41 123456789', - }, }, undefined, 'userId', ); } -export function ensureMachineUserExists(api: API, username: string): Cypress.Chainable { +export function ensureMachineUserExists(api: API, username: string) { return ensureItemExists( api, `${api.mgmtBaseURL}/users/_search`, @@ -41,7 +33,7 @@ export function ensureMachineUserExists(api: API, username: string): Cypress.Cha ); } -export function ensureUserDoesntExist(api: API, username: string): Cypress.Chainable { +export function ensureUserDoesntExist(api: API, username: string) { return ensureItemDoesntExist( api, `${api.mgmtBaseURL}/users/_search`, @@ -49,3 +41,31 @@ export function ensureUserDoesntExist(api: API, username: string): Cypress.Chain (user) => `${api.mgmtBaseURL}/users/${user.id}`, ); } + +export function createHumanUser(api: API, username: string, failOnStatusCode = true) { + return cy.request({ + method: 'POST', + url: `${api.mgmtBaseURL}/users/human`, + body: { + ...defaultHuman, + user_name: username, + }, + auth: { + bearer: api.token, + }, + failOnStatusCode: failOnStatusCode, + }); +} + +const defaultHuman = { + profile: { + first_name: 'e2efirstName', + last_name: 'e2elastName', + }, + email: { + email: 'e2e@email.ch', + }, + phone: { + phone: '+41 123456789', + }, +}; diff --git a/e2e/cypress/support/commands.ts b/e2e/cypress/support/commands.ts index fc8304a19..79658ea81 100644 --- a/e2e/cypress/support/commands.ts +++ b/e2e/cypress/support/commands.ts @@ -1,29 +1,8 @@ import 'cypress-wait-until'; -// -//namespace Cypress { -// interface Chainable { -// /** -// * Custom command that authenticates a user. -// * -// * @example cy.consolelogin('hodor', 'hodor1234') -// */ -// consolelogin(username: string, password: string): void -// } -//} -// -//Cypress.Commands.add('consolelogin', { prevSubject: false }, (username: string, password: string) => { -// -// window.sessionStorage.removeItem("zitadel:access_token") -// cy.visit(Cypress.config('baseUrl')/ui/console).then(() => { -// // fill the fields and push button -// cy.get('#loginName').type(username, { log: false }) -// cy.get('#submit-button').click() -// cy.get('#password').type(password, { log: false }) -// cy.get('#submit-button').click() -// cy.location('pathname', {timeout: 5 * 1000}).should('eq', '/'); -// }) -//}) -// +import { apiAuth, systemAuth } from './api/apiauth'; +import { API, SystemAPI } from './api/types'; +import { ensureQuotaIsRemoved, Unit } from './api/quota'; +import { instanceUnderTest } from './api/instances'; interface ShouldNotExistOptions { selector: string; @@ -46,7 +25,13 @@ declare global { /** * Custom command that waits until the selector finds zero elements. */ - shouldNotExist(options: ShouldNotExistOptions): Cypress.Chainable; + shouldNotExist(options?: ShouldNotExistOptions): Cypress.Chainable; + + /** + * Custom command that ensures a reliable testing context and returns it + */ + context(): Cypress.Chainable; + /** * Custom command that asserts success is printed after a change. */ @@ -55,6 +40,12 @@ declare global { } } +export interface Context { + api: API; + system: SystemAPI; + instanceId: number; +} + Cypress.Commands.add('clipboardMatches', { prevSubject: false }, (pattern: RegExp | string) => { /* doesn't work reliably return cy.window() @@ -106,3 +97,35 @@ Cypress.Commands.add('shouldConfirmSuccess', { prevSubject: false }, () => { cy.shouldNotExist({ selector: '.data-e2e-failure' }); cy.get('.data-e2e-success'); }); + +Cypress.Commands.add('context', { prevSubject: false }, () => { + return systemAuth().then((system) => { + return instanceUnderTest(system).then((instanceId) => { + return ensureQuotaIsRemoved( + { + system: system, + api: null, + instanceId: instanceId, + }, + Unit.AuthenticatedRequests, + ).then(() => { + return ensureQuotaIsRemoved( + { + system: system, + api: null, + instanceId: instanceId, + }, + Unit.ExecutionSeconds, + ).then(() => { + return apiAuth().then((api) => { + return { + system: system, + api: api, + instanceId: instanceId, + }; + }); + }); + }); + }); + }); +}); diff --git a/e2e/cypress/support/e2e.ts b/e2e/cypress/support/e2e.ts index 598ab5f0d..20c74362a 100644 --- a/e2e/cypress/support/e2e.ts +++ b/e2e/cypress/support/e2e.ts @@ -15,6 +15,4 @@ // Import commands.js using ES2015 syntax: import './commands'; - -// Alternatively you can use CommonJS syntax: -// require('./commands') +import './types'; diff --git a/e2e/cypress/support/index.ts b/e2e/cypress/support/index.ts deleted file mode 100644 index d2356297c..000000000 --- a/e2e/cypress/support/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -require('cypress-terminal-report/src/installLogsCollector')(); - -/** - * @type {Cypress.PluginConfig} - */ -module.exports = (on, config) => { - // `on` is used to hook into various events Cypress emits - // `config` is the resolved Cypress config -}; -//import './commands' diff --git a/e2e/cypress/support/login/users.ts b/e2e/cypress/support/login/users.ts index f02e350ae..e04bde8f8 100644 --- a/e2e/cypress/support/login/users.ts +++ b/e2e/cypress/support/login/users.ts @@ -82,8 +82,10 @@ export function login( onAuthenticated ? onAuthenticated() : null; + cy.visit('/'); + cy.get('[data-e2e=authenticated-welcome]', { - timeout: 10_000, + timeout: 50_000, }); }, { diff --git a/e2e/cypress/support/types.ts b/e2e/cypress/support/types.ts new file mode 100644 index 000000000..8fb29f538 --- /dev/null +++ b/e2e/cypress/support/types.ts @@ -0,0 +1,10 @@ +let webhookEventSchema = { + unit: 0, + id: '', + callURL: '', + periodStart: new Date(), + threshold: 0, + usage: 0, +}; + +export type ZITADELWebhookEvent = typeof webhookEventSchema; diff --git a/e2e/docker-compose-zitadel.yaml b/e2e/docker-compose-zitadel.yaml new file mode 100644 index 000000000..8b76f18ab --- /dev/null +++ b/e2e/docker-compose-zitadel.yaml @@ -0,0 +1,31 @@ +FirstInstance: + Org: + Human: + PasswordChangeRequired: false + +LogStore: + Access: + Database: + Enabled: true + Debounce: + MinFrequency: 0s + MaxBulkSize: 0 + Execution: + Database: + Enabled: true + Stdout: + Enabled: false + +Quotas: + Access: + ExhaustedCookieKey: "zitadel.quota.limiting" + ExhaustedCookieMaxAge: "60s" + +DefaultInstance: + LoginPolicy: + MfaInitSkipLifetime: 0 + +SystemAPIUsers: +- cypress: + KeyData: "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF6aStGRlNKTDdmNXl3NEtUd3pnTQpQMzRlUEd5Y20vTStrVDBNN1Y0Q2d4NVYzRWFESXZUUUtUTGZCYUVCNDV6YjlMdGpJWHpEdzByWFJvUzJoTzZ0CmgrQ1lRQ3ozS0N2aDA5QzBJenhaaUIySVMzSC9hVCs1Qng5RUZZK3ZuQWtaamNjYnlHNVlOUnZtdE9sbnZJZUkKSDdxWjB0RXdrUGZGNUdFWk5QSlB0bXkzVUdWN2lvZmRWUVMxeFJqNzMrYU13NXJ2SDREOElkeWlBQzNWZWtJYgpwdDBWajBTVVgzRHdLdG9nMzM3QnpUaVBrM2FYUkYwc2JGaFFvcWRKUkk4TnFnWmpDd2pxOXlmSTV0eXhZc3duCitKR3pIR2RIdlczaWRPRGxtd0V0NUsycGFzaVJJV0syT0dmcSt3MEVjbHRRSGFidXFFUGdabG1oQ2tSZE5maXgKQndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==" + diff --git a/e2e/docker-compose.yaml b/e2e/docker-compose.yaml index 8389f6130..39771ae99 100644 --- a/e2e/docker-compose.yaml +++ b/e2e/docker-compose.yaml @@ -2,20 +2,16 @@ version: '3.8' services: zitadel: + user: '$UID' restart: 'always' image: '${ZITADEL_IMAGE:-ghcr.io/zitadel/zitadel:latest}' - command: 'start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled' - environment: - ZITADEL_DATABASE_COCKROACH_HOST: db - ZITADEL_FIRSTINSTANCE_ORG_HUMAN_PASSWORDCHANGEREQUIRED: false - ZITADEL_DEFAULTINSTANCE_LOGINPOLICY_MFAINITSKIPLIFETIME: 0 + command: 'start-from-init --masterkey "MasterkeyNeedsToHave32Characters" --tlsMode disabled --config /zitadel.yaml --steps /zitadel.yaml' depends_on: db: condition: 'service_healthy' - ports: - - '8080:8080' - networks: - - zitadel + volumes: + - ./docker-compose-zitadel.yaml:/zitadel.yaml + network_mode: host db: restart: 'always' @@ -27,11 +23,7 @@ services: timeout: '30s' retries: 5 start_period: '20s' - ports: - - '9090:8080' - - '26257:26257' - networks: - - zitadel + network_mode: host prepare: image: node:18-alpine3.15 @@ -43,7 +35,7 @@ services: network_mode: host e2e: - image: cypress/included:10.9.0 + image: cypress/included:12.2.0 depends_on: zitadel: condition: 'service_started' diff --git a/e2e/package-lock.json b/e2e/package-lock.json index ca4708e71..1ed795dad 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -8,8 +8,11 @@ "name": "zitadel-e2e", "version": "0.0.0", "dependencies": { + "@types/pg": "^8.6.6", "cypress-wait-until": "^1.7.2", + "jsonwebtoken": "^8.5.1", "mochawesome": "^7.1.3", + "pg": "^8.8.0", "prettier": "^2.7.1", "typescript": "^4.8.4", "uuid": "^9.0.0", @@ -17,7 +20,7 @@ }, "devDependencies": { "@types/node": "^18.8.3", - "cypress": "^10.9.0" + "cypress": "^12.2.0" } }, "node_modules/@colors/colors": { @@ -121,8 +124,17 @@ "node_modules/@types/node": { "version": "18.8.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.8.3.tgz", - "integrity": "sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==", - "dev": true + "integrity": "sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==" + }, + "node_modules/@types/pg": { + "version": "8.6.6", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz", + "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } }, "node_modules/@types/sinonjs__fake-timers": { "version": "8.1.1", @@ -436,6 +448,19 @@ "node": "*" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/cachedir": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz", @@ -671,9 +696,9 @@ } }, "node_modules/cypress": { - "version": "10.9.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-10.9.0.tgz", - "integrity": "sha512-MjIWrRpc+bQM9U4kSSdATZWZ2hUqHGFEQTF7dfeZRa4MnalMtc88FIE49USWP2ZVtfy5WPBcgfBX+YorFqGElA==", + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.2.0.tgz", + "integrity": "sha512-kvl95ri95KK8mAy++tEU/wUgzAOMiIciZSL97LQvnOinb532m7dGvwN0mDSIGbOd71RREtmT9o4h088RjK5pKw==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -724,7 +749,7 @@ "cypress": "bin/cypress" }, "engines": { - "node": ">=12.0.0" + "node": "^14.0.0 || ^16.0.0 || >=18.0.0" } }, "node_modules/cypress-wait-until": { @@ -819,6 +844,14 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -1483,6 +1516,35 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=4", + "npm": ">=1.4.28" + } + }, + "node_modules/jsonwebtoken/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "bin": { + "semver": "bin/semver" + } + }, "node_modules/jsprim": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", @@ -1498,6 +1560,25 @@ "verror": "1.10.0" } }, + "node_modules/jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/lazy-ass": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz", @@ -1554,6 +1635,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, "node_modules/lodash.isempty": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", @@ -1564,11 +1655,26 @@ "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==" + }, "node_modules/lodash.isobject": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-3.0.2.tgz", "integrity": "sha512-3/Qptq2vr7WeJbB4KHUSKlq8Pl7ASXi3UG6CMbBm8WRtXi8+GHm7mKaU3urfpSEzWe2wCIChs6/sdocUsTKJiA==" }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, "node_modules/lodash.isstring": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", @@ -1577,8 +1683,7 @@ "node_modules/lodash.once": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", - "dev": true + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" }, "node_modules/log-symbols": { "version": "4.1.0", @@ -2004,6 +2109,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -2042,6 +2152,80 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", "dev": true }, + "node_modules/pg": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz", + "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.5.2", + "pg-protocol": "^1.5.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz", + "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -2063,6 +2247,41 @@ "node": ">=0.10.0" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prettier": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", @@ -2314,6 +2533,14 @@ "node": ">=8" } }, + "node_modules/split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -2614,6 +2841,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "engines": { + "node": ">=0.4" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -2790,8 +3025,17 @@ "@types/node": { "version": "18.8.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.8.3.tgz", - "integrity": "sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==", - "dev": true + "integrity": "sha512-0os9vz6BpGwxGe9LOhgP/ncvYN5Tx1fNcd2TM3rD/aCGBkysb+ZWpXEocG24h6ZzOi13+VB8HndAQFezsSOw1w==" + }, + "@types/pg": { + "version": "8.6.6", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.6.tgz", + "integrity": "sha512-O2xNmXebtwVekJDD+02udOncjVcMZQuTEQEMpKJ0ZRf5E7/9JJX3izhKUcUifBkyKpljyUM6BTgy2trmviKlpw==", + "requires": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } }, "@types/sinonjs__fake-timers": { "version": "8.1.1", @@ -3018,6 +3262,16 @@ "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", "dev": true }, + "buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "cachedir": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.3.0.tgz", @@ -3191,9 +3445,9 @@ } }, "cypress": { - "version": "10.9.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-10.9.0.tgz", - "integrity": "sha512-MjIWrRpc+bQM9U4kSSdATZWZ2hUqHGFEQTF7dfeZRa4MnalMtc88FIE49USWP2ZVtfy5WPBcgfBX+YorFqGElA==", + "version": "12.2.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.2.0.tgz", + "integrity": "sha512-kvl95ri95KK8mAy++tEU/wUgzAOMiIciZSL97LQvnOinb532m7dGvwN0mDSIGbOd71RREtmT9o4h088RjK5pKw==", "dev": true, "requires": { "@cypress/request": "^2.88.10", @@ -3308,6 +3562,14 @@ "safer-buffer": "^2.1.0" } }, + "ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "requires": { + "safe-buffer": "^5.0.1" + } + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -3791,6 +4053,30 @@ "universalify": "^2.0.0" } }, + "jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "requires": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, "jsprim": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-2.0.2.tgz", @@ -3803,6 +4089,25 @@ "verror": "1.10.0" } }, + "jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "requires": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, "lazy-ass": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/lazy-ass/-/lazy-ass-1.6.0.tgz", @@ -3839,6 +4144,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, "lodash.isempty": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.isempty/-/lodash.isempty-4.4.0.tgz", @@ -3849,11 +4164,26 @@ "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" }, + "lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" + }, + "lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==" + }, "lodash.isobject": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/lodash.isobject/-/lodash.isobject-3.0.2.tgz", "integrity": "sha512-3/Qptq2vr7WeJbB4KHUSKlq8Pl7ASXi3UG6CMbBm8WRtXi8+GHm7mKaU3urfpSEzWe2wCIChs6/sdocUsTKJiA==" }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, "lodash.isstring": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", @@ -3862,8 +4192,7 @@ "lodash.once": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", - "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", - "dev": true + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" }, "log-symbols": { "version": "4.1.0", @@ -4178,6 +4507,11 @@ "aggregate-error": "^3.0.0" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -4207,6 +4541,61 @@ "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==", "dev": true }, + "pg": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.8.0.tgz", + "integrity": "sha512-UXYN0ziKj+AeNNP7VDMwrehpACThH7LUl/p8TDFpEUuSejCUIwGSfxpHsPvtM6/WXFy6SU4E5RG4IJV/TZAGjw==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.5.0", + "pg-pool": "^3.5.2", + "pg-protocol": "^1.5.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-connection-string": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", + "integrity": "sha512-r5o/V/ORTA6TmUnyWZR9nCj1klXCO2CEKNRlVuJptZe85QuhFayC7WeMic7ndayT5IRIR0S0xFxFi2ousartlQ==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-pool": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.5.2.tgz", + "integrity": "sha512-His3Fh17Z4eg7oANLob6ZvH8xIVen3phEZh2QuyrIl4dQSDVEabNducv6ysROKpDNPSD+12tONZVWfSgMvDD9w==", + "requires": {} + }, + "pg-protocol": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.5.0.tgz", + "integrity": "sha512-muRttij7H8TqRNu/DxrAJQITO4Ac7RmX3Klyr/9mJEOBeIpgnF8f9jAfRz5d3XwQZl5qBjF9gLsUtMPJE0vezQ==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + } + }, "picomatch": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", @@ -4219,6 +4608,29 @@ "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", "dev": true }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, "prettier": { "version": "2.7.1", "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", @@ -4405,6 +4817,11 @@ "is-fullwidth-code-point": "^3.0.0" } }, + "split2": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" + }, "sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -4618,6 +5035,11 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/e2e/package.json b/e2e/package.json index a8ef8c3bd..1cbe649fb 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -11,8 +11,11 @@ }, "private": true, "dependencies": { + "@types/pg": "^8.6.6", "cypress-wait-until": "^1.7.2", + "jsonwebtoken": "^8.5.1", "mochawesome": "^7.1.3", + "pg": "^8.8.0", "prettier": "^2.7.1", "typescript": "^4.8.4", "uuid": "^9.0.0", @@ -20,6 +23,6 @@ }, "devDependencies": { "@types/node": "^18.8.3", - "cypress": "^10.9.0" + "cypress": "^12.2.0" } } \ No newline at end of file diff --git a/go.mod b/go.mod index a58ce9672..1ae71ea18 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/VictoriaMetrics/fastcache v1.8.0 github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/allegro/bigcache v1.2.1 + github.com/benbjohnson/clock v1.2.0 github.com/boombuler/barcode v1.0.1 github.com/cockroachdb/cockroach-go/v2 v2.2.18 github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6 @@ -170,7 +171,7 @@ require ( github.com/rs/xid v1.2.1 // indirect github.com/russellhaering/goxmldsig v1.2.0 // indirect github.com/satori/go.uuid v1.2.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.0 github.com/spf13/afero v1.8.1 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect diff --git a/internal/actions/actions.go b/internal/actions/actions.go index 9cdb13765..34d745d2c 100644 --- a/internal/actions/actions.go +++ b/internal/actions/actions.go @@ -6,6 +6,8 @@ import ( "fmt" "github.com/dop251/goja_nodejs/require" + "github.com/sirupsen/logrus" + z_errs "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/query" ) @@ -14,15 +16,45 @@ type Config struct { HTTP HTTPConfig } -var ( - ErrHalt = errors.New("interrupt") -) +var ErrHalt = errors.New("interrupt") type jsAction func(fields, fields) error -func Run(ctx context.Context, ctxParam contextFields, apiParam apiFields, script, name string, opts ...Option) error { - config, err := prepareRun(ctx, ctxParam, apiParam, script, opts) - if err != nil { +const ( + actionStartedMessage = "action run started" + actionSucceededMessage = "action run succeeded" +) + +func actionFailedMessage(err error) string { + return fmt.Sprintf("action run failed: %s", err.Error()) +} + +func Run(ctx context.Context, ctxParam contextFields, apiParam apiFields, script, name string, opts ...Option) (err error) { + config := newRunConfig(ctx, append(opts, withLogger(ctx))...) + if config.functionTimeout == 0 { + return z_errs.ThrowInternal(nil, "ACTIO-uCpCx", "Errrors.Internal") + } + + remaining := logstoreService.Limit(ctx, config.instanceID) + config.cutTimeouts(remaining) + + config.logger.Log(actionStartedMessage) + if remaining != nil && *remaining == 0 { + return z_errs.ThrowResourceExhausted(nil, "ACTIO-f19Ii", "Errors.Quota.Execution.Exhausted") + } + + defer func() { + if err != nil { + config.logger.log(actionFailedMessage(err), logrus.ErrorLevel, true) + } else { + config.logger.log(actionSucceededMessage, logrus.InfoLevel, true) + } + if config.allowedToFail { + err = nil + } + }() + + if err := executeScript(config, ctxParam, apiParam, script); err != nil { return err } @@ -31,12 +63,11 @@ func Run(ctx context.Context, ctxParam contextFields, apiParam apiFields, script if jsFn == nil { return errors.New("function not found") } - err = config.vm.ExportTo(jsFn, &fn) - if err != nil { + if err := config.vm.ExportTo(jsFn, &fn); err != nil { return err } - t := config.Start() + t := config.StartFunction() defer func() { t.Stop() }() @@ -44,12 +75,8 @@ func Run(ctx context.Context, ctxParam contextFields, apiParam apiFields, script return executeFn(config, fn) } -func prepareRun(ctx context.Context, ctxParam contextFields, apiParam apiFields, script string, opts []Option) (config *runConfig, err error) { - config = newRunConfig(ctx, opts...) - if config.timeout == 0 { - return nil, z_errs.ThrowInternal(nil, "ACTIO-uCpCx", "Errrors.Internal") - } - t := config.Prepare() +func executeScript(config *runConfig, ctxParam contextFields, apiParam apiFields, script string) (err error) { + t := config.StartScript() defer func() { t.Stop() }() @@ -67,7 +94,6 @@ func prepareRun(ctx context.Context, ctxParam contextFields, apiParam apiFields, for name, loader := range config.modules { registry.RegisterNativeModule(name, loader) } - // overload error if function panics defer func() { r := recover() @@ -76,29 +102,31 @@ func prepareRun(ctx context.Context, ctxParam contextFields, apiParam apiFields, return } }() + _, err = config.vm.RunString(script) - return config, err + return err } func executeFn(config *runConfig, fn jsAction) (err error) { defer func() { r := recover() - if r != nil && !config.allowedToFail { - var ok bool - if err, ok = r.(error); ok { - return - } - - e, ok := r.(string) - if ok { - err = errors.New(e) - return - } - err = fmt.Errorf("unknown error occured: %v", r) + if r == nil { + return } + var ok bool + if err, ok = r.(error); ok { + return + } + + e, ok := r.(string) + if ok { + err = errors.New(e) + return + } + err = fmt.Errorf("unknown error occurred: %v", r) }() - err = fn(config.ctxParam.fields, config.apiParam.fields) - if err != nil && !config.allowedToFail { + + if err = fn(config.ctxParam.fields, config.apiParam.fields); err != nil { return err } return nil diff --git a/internal/actions/actions_test.go b/internal/actions/actions_test.go index bf774cd1c..1287546a9 100644 --- a/internal/actions/actions_test.go +++ b/internal/actions/actions_test.go @@ -7,9 +7,11 @@ import ( "time" "github.com/dop251/goja" + "github.com/zitadel/zitadel/internal/logstore" ) func TestRun(t *testing.T) { + SetLogstoreService(logstore.New(nil, nil, nil)) type args struct { timeout time.Duration api apiFields diff --git a/internal/actions/config.go b/internal/actions/config.go index fd53e9e75..1d0ff78e7 100644 --- a/internal/actions/config.go +++ b/internal/actions/config.go @@ -23,13 +23,14 @@ func WithAllowedToFail() Option { type runConfig struct { allowedToFail bool - timeout, - prepareTimeout time.Duration - modules map[string]require.ModuleLoader - - vm *goja.Runtime - ctxParam *ctxConfig - apiParam *apiConfig + functionTimeout, + scriptTimeout time.Duration + modules map[string]require.ModuleLoader + logger *logger + instanceID string + vm *goja.Runtime + ctxParam *ctxConfig + apiParam *apiConfig } func newRunConfig(ctx context.Context, opts ...Option) *runConfig { @@ -42,10 +43,10 @@ func newRunConfig(ctx context.Context, opts ...Option) *runConfig { vm.SetFieldNameMapper(goja.UncapFieldNameMapper()) config := &runConfig{ - timeout: time.Until(deadline), - prepareTimeout: maxPrepareTimeout, - modules: map[string]require.ModuleLoader{}, - vm: vm, + functionTimeout: time.Until(deadline), + scriptTimeout: maxPrepareTimeout, + modules: map[string]require.ModuleLoader{}, + vm: vm, ctxParam: &ctxConfig{ FieldConfig: FieldConfig{ Runtime: vm, @@ -64,23 +65,37 @@ func newRunConfig(ctx context.Context, opts ...Option) *runConfig { opt(config) } - if config.prepareTimeout > config.timeout { - config.prepareTimeout = config.timeout + if config.scriptTimeout > config.functionTimeout { + config.scriptTimeout = config.functionTimeout } return config } -func (c *runConfig) Start() *time.Timer { +func (c *runConfig) StartFunction() *time.Timer { c.vm.ClearInterrupt() - return time.AfterFunc(c.timeout, func() { + return time.AfterFunc(c.functionTimeout, func() { c.vm.Interrupt(ErrHalt) }) } -func (c *runConfig) Prepare() *time.Timer { +func (c *runConfig) StartScript() *time.Timer { c.vm.ClearInterrupt() - return time.AfterFunc(c.prepareTimeout, func() { + return time.AfterFunc(c.scriptTimeout, func() { c.vm.Interrupt(ErrHalt) }) } + +func (c *runConfig) cutTimeouts(remainingSeconds *uint64) { + if remainingSeconds == nil { + return + } + + remainingDur := time.Duration(*remainingSeconds) * time.Second + if c.functionTimeout > remainingDur { + c.functionTimeout = remainingDur + } + if c.scriptTimeout > remainingDur { + c.scriptTimeout = remainingDur + } +} diff --git a/internal/actions/fields_test.go b/internal/actions/fields_test.go index 846b9cecb..93862cfb3 100644 --- a/internal/actions/fields_test.go +++ b/internal/actions/fields_test.go @@ -5,9 +5,11 @@ import ( "testing" "github.com/dop251/goja" + "github.com/zitadel/zitadel/internal/logstore" ) func TestSetFields(t *testing.T) { + SetLogstoreService(logstore.New(nil, nil, nil)) primitveFn := func(a string) { fmt.Println(a) } complexFn := func(*FieldConfig) interface{} { return primitveFn diff --git a/internal/actions/http_module_test.go b/internal/actions/http_module_test.go index 9ebb5aed5..176f789e2 100644 --- a/internal/actions/http_module_test.go +++ b/internal/actions/http_module_test.go @@ -11,9 +11,11 @@ import ( "github.com/dop251/goja" "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/logstore" ) func Test_isHostBlocked(t *testing.T) { + SetLogstoreService(logstore.New(nil, nil, nil)) var denyList = []AddressChecker{ mustNewIPChecker(t, "192.168.5.0/24"), mustNewIPChecker(t, "127.0.0.1"), diff --git a/internal/actions/log_module.go b/internal/actions/log_module.go index bb9b20e01..1d2e28adf 100644 --- a/internal/actions/log_module.go +++ b/internal/actions/log_module.go @@ -1,30 +1,83 @@ package actions import ( - "github.com/zitadel/logging" + "context" + "time" "github.com/dop251/goja" "github.com/dop251/goja_nodejs/console" + "github.com/sirupsen/logrus" + + "github.com/zitadel/zitadel/internal/api/authz" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/logstore/emitters/execution" ) -var ServerLog *logrus +var ( + logstoreService *logstore.Service + _ console.Printer = (*logger)(nil) +) -type logrus struct{} - -func (*logrus) Log(s string) { - logging.WithFields("message", s).Info("log from action") -} -func (*logrus) Warn(s string) { - logging.WithFields("message", s).Info("warn from action") -} -func (*logrus) Error(s string) { - logging.WithFields("message", s).Info("error from action") +func SetLogstoreService(svc *logstore.Service) { + logstoreService = svc } -func WithLogger(logger console.Printer) Option { +type logger struct { + ctx context.Context + started time.Time + instanceID string +} + +// newLogger returns a *logger instance that should only be used for a single action run. +// The first log call sets the started field for subsequent log calls +func newLogger(ctx context.Context, instanceID string) *logger { + return &logger{ + ctx: ctx, + started: time.Time{}, + instanceID: instanceID, + } +} + +func (l *logger) Log(msg string) { + l.log(msg, logrus.InfoLevel, false) +} + +func (l *logger) Warn(msg string) { + l.log(msg, logrus.WarnLevel, false) +} + +func (l *logger) Error(msg string) { + l.log(msg, logrus.ErrorLevel, false) +} + +func (l *logger) log(msg string, level logrus.Level, last bool) { + ts := time.Now() + if l.started.IsZero() { + l.started = ts + } + + record := &execution.Record{ + LogDate: ts, + InstanceID: l.instanceID, + Message: msg, + LogLevel: level, + } + + if last { + record.Took = ts.Sub(l.started) + } + + logstoreService.Handle(l.ctx, record) +} + +func withLogger(ctx context.Context) Option { + instance := authz.GetInstance(ctx) + instanceID := instance.InstanceID() return func(c *runConfig) { + c.logger = newLogger(ctx, instanceID) + c.instanceID = instanceID c.modules["zitadel/log"] = func(runtime *goja.Runtime, module *goja.Object) { - console.RequireWithPrinter(logger)(runtime, module) + console.RequireWithPrinter(c.logger)(runtime, module) } } } diff --git a/internal/api/api.go b/internal/api/api.go index 8e1d9a9e2..ec51d7fff 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -16,6 +16,7 @@ import ( http_util "github.com/zitadel/zitadel/internal/api/http" "github.com/zitadel/zitadel/internal/api/ui/login" "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/logstore" "github.com/zitadel/zitadel/internal/query" "github.com/zitadel/zitadel/internal/telemetry/metrics" "github.com/zitadel/zitadel/internal/telemetry/tracing" @@ -36,7 +37,18 @@ type health interface { Instance(ctx context.Context, shouldTriggerBulk bool) (*query.Instance, error) } -func New(port uint16, router *mux.Router, queries *query.Queries, verifier *internal_authz.TokenVerifier, authZ internal_authz.Config, externalSecure bool, tlsConfig *tls.Config, http2HostName, http1HostName string) *API { +func New( + port uint16, + router *mux.Router, + queries *query.Queries, + verifier *internal_authz.TokenVerifier, + authZ internal_authz.Config, + externalSecure bool, + tlsConfig *tls.Config, + http2HostName, + http1HostName string, + accessSvc *logstore.Service, +) *API { api := &API{ port: port, verifier: verifier, @@ -45,7 +57,8 @@ func New(port uint16, router *mux.Router, queries *query.Queries, verifier *inte externalSecure: externalSecure, http1HostName: http1HostName, } - api.grpcServer = server.CreateServer(api.verifier, authZ, queries, http2HostName, tlsConfig) + + api.grpcServer = server.CreateServer(api.verifier, authZ, queries, http2HostName, tlsConfig, accessSvc) api.routeGRPC() api.RegisterHandler("/debug", api.healthHandler()) diff --git a/internal/api/assets/asset.go b/internal/api/assets/asset.go index 8117ab696..815d1a17c 100644 --- a/internal/api/assets/asset.go +++ b/internal/api/assets/asset.go @@ -82,7 +82,7 @@ func DefaultErrorHandler(w http.ResponseWriter, r *http.Request, err error, code http.Error(w, err.Error(), code) } -func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, instanceInterceptor, assetCacheInterceptor func(handler http.Handler) http.Handler) http.Handler { +func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authConfig authz.Config, idGenerator id.Generator, storage static.Storage, queries *query.Queries, instanceInterceptor, assetCacheInterceptor, accessInterceptor func(handler http.Handler) http.Handler) http.Handler { h := &Handler{ commands: commands, errorHandler: DefaultErrorHandler, @@ -94,7 +94,7 @@ func NewHandler(commands *command.Commands, verifier *authz.TokenVerifier, authC verifier.RegisterServer("Assets-API", "assets", AssetsService_AuthMethods) router := mux.NewRouter() - router.Use(instanceInterceptor, assetCacheInterceptor) + router.Use(instanceInterceptor, assetCacheInterceptor, accessInterceptor) RegisterRoutes(router, h) router.PathPrefix("/{owner}").Methods("GET").HandlerFunc(DownloadHandleFunc(h, h.GetFile())) return http_util.CopyHeadersToContext(http_mw.CORSInterceptor(router)) diff --git a/internal/api/authz/instance.go b/internal/api/authz/instance.go index e490c4cdc..217fb9d9b 100644 --- a/internal/api/authz/instance.go +++ b/internal/api/authz/instance.go @@ -23,7 +23,8 @@ type Instance interface { } type InstanceVerifier interface { - InstanceByHost(context.Context, string) (Instance, error) + InstanceByHost(ctx context.Context, host string) (Instance, error) + InstanceByID(ctx context.Context) (Instance, error) } type instance struct { diff --git a/internal/api/grpc/errors/caos_errors.go b/internal/api/grpc/errors/caos_errors.go index 2e8117c8c..a1d6c2506 100644 --- a/internal/api/grpc/errors/caos_errors.go +++ b/internal/api/grpc/errors/caos_errors.go @@ -55,6 +55,8 @@ func ExtractCaosError(err error) (c codes.Code, msg, id string, ok bool) { return codes.Unavailable, caosErr.GetMessage(), caosErr.GetID(), true case *caos_errs.UnimplementedError: return codes.Unimplemented, caosErr.GetMessage(), caosErr.GetID(), true + case *caos_errs.ResourceExhaustedError: + return codes.ResourceExhausted, caosErr.GetMessage(), caosErr.GetID(), true default: return codes.Unknown, err.Error(), "", false } diff --git a/internal/api/grpc/errors/caos_errors_test.go b/internal/api/grpc/errors/caos_errors_test.go index 13c6c1b73..3a14f1bbf 100644 --- a/internal/api/grpc/errors/caos_errors_test.go +++ b/internal/api/grpc/errors/caos_errors_test.go @@ -136,6 +136,14 @@ func Test_Extract(t *testing.T) { "id", true, }, + { + "exhausted", + args{caos_errs.ThrowResourceExhausted(nil, "id", "exhausted")}, + codes.ResourceExhausted, + "exhausted", + "id", + true, + }, { "unknown", args{errors.New("unknown")}, diff --git a/internal/api/grpc/server/middleware/access_interceptor.go b/internal/api/grpc/server/middleware/access_interceptor.go new file mode 100644 index 000000000..2b3cd43df --- /dev/null +++ b/internal/api/grpc/server/middleware/access_interceptor.go @@ -0,0 +1,55 @@ +package middleware + +import ( + "context" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + + "github.com/zitadel/zitadel/internal/api/authz" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/logstore/emitters/access" + "github.com/zitadel/zitadel/internal/telemetry/tracing" +) + +func AccessStorageInterceptor(svc *logstore.Service) grpc.UnaryServerInterceptor { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) { + if !svc.Enabled() { + return handler(ctx, req) + } + + reqMd, _ := metadata.FromIncomingContext(ctx) + + resp, handlerErr := handler(ctx, req) + + interceptorCtx, span := tracing.NewServerInterceptorSpan(ctx) + defer func() { span.EndWithError(err) }() + + var respStatus uint32 + grpcStatus, ok := status.FromError(handlerErr) + if ok { + respStatus = uint32(grpcStatus.Code()) + } + + resMd, _ := metadata.FromOutgoingContext(ctx) + instance := authz.GetInstance(ctx) + + record := &access.Record{ + LogDate: time.Now(), + Protocol: access.GRPC, + RequestURL: info.FullMethod, + ResponseStatus: respStatus, + RequestHeaders: reqMd, + ResponseHeaders: resMd, + InstanceID: instance.InstanceID(), + ProjectID: instance.ProjectID(), + RequestedDomain: instance.RequestedDomain(), + RequestedHost: instance.RequestedHost(), + } + + svc.Handle(interceptorCtx, record) + return resp, handlerErr + } +} diff --git a/internal/api/grpc/server/middleware/instance_interceptor.go b/internal/api/grpc/server/middleware/instance_interceptor.go index 799382437..77302fdf7 100644 --- a/internal/api/grpc/server/middleware/instance_interceptor.go +++ b/internal/api/grpc/server/middleware/instance_interceptor.go @@ -2,7 +2,7 @@ package middleware import ( "context" - "errors" + errs "errors" "fmt" "strings" @@ -14,7 +14,7 @@ import ( "google.golang.org/grpc/status" "github.com/zitadel/zitadel/internal/api/authz" - caos_errors "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/i18n" "github.com/zitadel/zitadel/internal/telemetry/tracing" ) @@ -23,27 +23,36 @@ const ( HTTP1Host = "x-zitadel-http1-host" ) -type InstanceVerifier interface { - GetInstance(ctx context.Context) -} - -func InstanceInterceptor(verifier authz.InstanceVerifier, headerName string, ignoredServices ...string) grpc.UnaryServerInterceptor { +func InstanceInterceptor(verifier authz.InstanceVerifier, headerName string, explicitInstanceIdServices ...string) grpc.UnaryServerInterceptor { translator, err := newZitadelTranslator(language.English) logging.OnError(err).Panic("unable to get translator") return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - return setInstance(ctx, req, info, handler, verifier, headerName, translator, ignoredServices...) + return setInstance(ctx, req, info, handler, verifier, headerName, translator, explicitInstanceIdServices...) } } -func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, headerName string, translator *i18n.Translator, ignoredServices ...string) (_ interface{}, err error) { +func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, headerName string, translator *i18n.Translator, idFromRequestsServices ...string) (_ interface{}, err error) { interceptorCtx, span := tracing.NewServerInterceptorSpan(ctx) defer func() { span.EndWithError(err) }() - for _, service := range ignoredServices { + for _, service := range idFromRequestsServices { if !strings.HasPrefix(service, "/") { service = "/" + service } if strings.HasPrefix(info.FullMethod, service) { - return handler(ctx, req) + withInstanceIDProperty, ok := req.(interface{ GetInstanceId() string }) + if !ok { + return handler(ctx, req) + } + ctx = authz.WithInstanceID(ctx, withInstanceIDProperty.GetInstanceId()) + instance, err := verifier.InstanceByID(ctx) + if err != nil { + notFoundErr := new(errors.NotFoundError) + if errs.As(err, ¬FoundErr) { + notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil) + } + return nil, status.Error(codes.NotFound, err.Error()) + } + return handler(authz.WithInstance(ctx, instance), req) } } @@ -53,9 +62,9 @@ func setInstance(ctx context.Context, req interface{}, info *grpc.UnaryServerInf } instance, err := verifier.InstanceByHost(interceptorCtx, host) if err != nil { - caosErr := new(caos_errors.NotFoundError) - if errors.As(err, &caosErr) { - caosErr.Message = translator.LocalizeFromCtx(ctx, caosErr.GetMessage(), nil) + notFoundErr := new(errors.NotFoundError) + if errs.As(err, ¬FoundErr) { + notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil) } return nil, status.Error(codes.NotFound, err.Error()) } diff --git a/internal/api/grpc/server/middleware/instance_interceptor_test.go b/internal/api/grpc/server/middleware/instance_interceptor_test.go index 730908721..513dfe96c 100644 --- a/internal/api/grpc/server/middleware/instance_interceptor_test.go +++ b/internal/api/grpc/server/middleware/instance_interceptor_test.go @@ -153,13 +153,15 @@ type mockInstanceVerifier struct { host string } -func (m *mockInstanceVerifier) InstanceByHost(ctx context.Context, host string) (authz.Instance, error) { +func (m *mockInstanceVerifier) InstanceByHost(_ context.Context, host string) (authz.Instance, error) { if host != m.host { return nil, fmt.Errorf("invalid host") } return &mockInstance{}, nil } +func (m *mockInstanceVerifier) InstanceByID(context.Context) (authz.Instance, error) { return nil, nil } + type mockInstance struct{} func (m *mockInstance) InstanceID() string { diff --git a/internal/api/grpc/server/middleware/quota_interceptor.go b/internal/api/grpc/server/middleware/quota_interceptor.go new file mode 100644 index 000000000..08ce38907 --- /dev/null +++ b/internal/api/grpc/server/middleware/quota_interceptor.go @@ -0,0 +1,46 @@ +package middleware + +import ( + "context" + "strings" + + "google.golang.org/grpc" + + "github.com/zitadel/zitadel/internal/api/authz" + "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/telemetry/tracing" +) + +func QuotaExhaustedInterceptor(svc *logstore.Service, ignoreService ...string) grpc.UnaryServerInterceptor { + + prunedIgnoredServices := make([]string, len(ignoreService)) + for idx, service := range ignoreService { + if !strings.HasPrefix(service, "/") { + service = "/" + service + } + prunedIgnoredServices[idx] = service + } + + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (_ interface{}, err error) { + if !svc.Enabled() { + return handler(ctx, req) + } + interceptorCtx, span := tracing.NewServerInterceptorSpan(ctx) + defer func() { span.EndWithError(err) }() + + for _, service := range prunedIgnoredServices { + if strings.HasPrefix(info.FullMethod, service) { + return handler(ctx, req) + } + } + + instance := authz.GetInstance(ctx) + remaining := svc.Limit(interceptorCtx, instance.InstanceID()) + if remaining != nil && *remaining == 0 { + return nil, errors.ThrowResourceExhausted(nil, "QUOTA-vjAy8", "Quota.Access.Exhausted") + } + span.End() + return handler(ctx, req) + } +} diff --git a/internal/api/grpc/server/server.go b/internal/api/grpc/server/server.go index c077d8f56..b81d77d6c 100644 --- a/internal/api/grpc/server/server.go +++ b/internal/api/grpc/server/server.go @@ -2,7 +2,6 @@ package server import ( "crypto/tls" - grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -10,6 +9,7 @@ import ( "github.com/zitadel/zitadel/internal/api/authz" grpc_api "github.com/zitadel/zitadel/internal/api/grpc" "github.com/zitadel/zitadel/internal/api/grpc/server/middleware" + "github.com/zitadel/zitadel/internal/logstore" "github.com/zitadel/zitadel/internal/query" "github.com/zitadel/zitadel/internal/telemetry/metrics" system_pb "github.com/zitadel/zitadel/pkg/grpc/system" @@ -23,7 +23,14 @@ type Server interface { AuthMethods() authz.MethodMapping } -func CreateServer(verifier *authz.TokenVerifier, authConfig authz.Config, queries *query.Queries, hostHeaderName string, tlsConfig *tls.Config) *grpc.Server { +func CreateServer( + verifier *authz.TokenVerifier, + authConfig authz.Config, + queries *query.Queries, + hostHeaderName string, + tlsConfig *tls.Config, + accessSvc *logstore.Service, +) *grpc.Server { metricTypes := []metrics.MetricType{metrics.MetricTypeTotalCount, metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode} serverOptions := []grpc.ServerOption{ grpc.UnaryInterceptor( @@ -33,10 +40,12 @@ func CreateServer(verifier *authz.TokenVerifier, authConfig authz.Config, querie middleware.NoCacheInterceptor(), middleware.ErrorHandler(), middleware.InstanceInterceptor(queries, hostHeaderName, system_pb.SystemService_MethodPrefix), + middleware.AccessStorageInterceptor(accessSvc), middleware.AuthorizationInterceptor(verifier, authConfig), middleware.TranslationHandler(), middleware.ValidationHandler(), middleware.ServiceHandler(), + middleware.QuotaExhaustedInterceptor(accessSvc, system_pb.SystemService_MethodPrefix), ), ), } diff --git a/internal/api/grpc/system/instance.go b/internal/api/grpc/system/instance.go index 822c618bd..1d711062c 100644 --- a/internal/api/grpc/system/instance.go +++ b/internal/api/grpc/system/instance.go @@ -31,7 +31,6 @@ func (s *Server) ListInstances(ctx context.Context, req *system_pb.ListInstances } func (s *Server) GetInstance(ctx context.Context, req *system_pb.GetInstanceRequest) (*system_pb.GetInstanceResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) instance, err := s.query.Instance(ctx, true) if err != nil { return nil, err @@ -53,7 +52,6 @@ func (s *Server) AddInstance(ctx context.Context, req *system_pb.AddInstanceRequ } func (s *Server) UpdateInstance(ctx context.Context, req *system_pb.UpdateInstanceRequest) (*system_pb.UpdateInstanceResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) details, err := s.command.UpdateInstance(ctx, req.InstanceName) if err != nil { return nil, err @@ -86,7 +84,6 @@ func (s *Server) CreateInstance(ctx context.Context, req *system_pb.CreateInstan } func (s *Server) RemoveInstance(ctx context.Context, req *system_pb.RemoveInstanceRequest) (*system_pb.RemoveInstanceResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) details, err := s.command.RemoveInstance(ctx, req.InstanceId) if err != nil { return nil, err @@ -97,7 +94,6 @@ func (s *Server) RemoveInstance(ctx context.Context, req *system_pb.RemoveInstan } func (s *Server) ListIAMMembers(ctx context.Context, req *system_pb.ListIAMMembersRequest) (*system_pb.ListIAMMembersResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) queries, err := ListIAMMembersRequestToQuery(req) if err != nil { return nil, err @@ -139,7 +135,6 @@ func (s *Server) ExistsDomain(ctx context.Context, req *system_pb.ExistsDomainRe } func (s *Server) ListDomains(ctx context.Context, req *system_pb.ListDomainsRequest) (*system_pb.ListDomainsResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) queries, err := ListInstanceDomainsRequestToModel(req) if err != nil { return nil, err @@ -156,8 +151,6 @@ func (s *Server) ListDomains(ctx context.Context, req *system_pb.ListDomainsRequ } func (s *Server) AddDomain(ctx context.Context, req *system_pb.AddDomainRequest) (*system_pb.AddDomainResponse, error) { - //TODO: should be solved in interceptor - ctx = authz.WithInstanceID(ctx, req.InstanceId) instance, err := s.query.Instance(ctx, true) if err != nil { return nil, err @@ -174,7 +167,6 @@ func (s *Server) AddDomain(ctx context.Context, req *system_pb.AddDomainRequest) } func (s *Server) RemoveDomain(ctx context.Context, req *system_pb.RemoveDomainRequest) (*system_pb.RemoveDomainResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) details, err := s.command.RemoveInstanceDomain(ctx, req.Domain) if err != nil { return nil, err @@ -185,7 +177,6 @@ func (s *Server) RemoveDomain(ctx context.Context, req *system_pb.RemoveDomainRe } func (s *Server) SetPrimaryDomain(ctx context.Context, req *system_pb.SetPrimaryDomainRequest) (*system_pb.SetPrimaryDomainResponse, error) { - ctx = authz.WithInstanceID(ctx, req.InstanceId) details, err := s.command.SetPrimaryInstanceDomain(ctx, req.Domain) if err != nil { return nil, err diff --git a/internal/api/grpc/system/quota.go b/internal/api/grpc/system/quota.go new file mode 100644 index 000000000..745c449b0 --- /dev/null +++ b/internal/api/grpc/system/quota.go @@ -0,0 +1,32 @@ +package system + +import ( + "context" + + "github.com/zitadel/zitadel/internal/api/grpc/object" + "github.com/zitadel/zitadel/pkg/grpc/system" + system_pb "github.com/zitadel/zitadel/pkg/grpc/system" +) + +func (s *Server) AddQuota(ctx context.Context, req *system.AddQuotaRequest) (*system.AddQuotaResponse, error) { + details, err := s.command.AddQuota( + ctx, + instanceQuotaPbToCommand(req), + ) + if err != nil { + return nil, err + } + return &system_pb.AddQuotaResponse{ + Details: object.AddToDetailsPb(details.Sequence, details.EventDate, details.ResourceOwner), + }, nil +} + +func (s *Server) RemoveQuota(ctx context.Context, req *system.RemoveQuotaRequest) (*system.RemoveQuotaResponse, error) { + details, err := s.command.RemoveQuota(ctx, instanceQuotaUnitPbToCommand(req.Unit)) + if err != nil { + return nil, err + } + return &system_pb.RemoveQuotaResponse{ + Details: object.ChangeToDetailsPb(details.Sequence, details.EventDate, details.ResourceOwner), + }, nil +} diff --git a/internal/api/grpc/system/quota_converter.go b/internal/api/grpc/system/quota_converter.go new file mode 100644 index 000000000..5be57b43d --- /dev/null +++ b/internal/api/grpc/system/quota_converter.go @@ -0,0 +1,43 @@ +package system + +import ( + "github.com/zitadel/zitadel/internal/command" + "github.com/zitadel/zitadel/pkg/grpc/quota" + "github.com/zitadel/zitadel/pkg/grpc/system" +) + +func instanceQuotaPbToCommand(req *system.AddQuotaRequest) *command.AddQuota { + return &command.AddQuota{ + Unit: instanceQuotaUnitPbToCommand(req.Unit), + From: req.From.AsTime(), + ResetInterval: req.ResetInterval.AsDuration(), + Amount: req.Amount, + Limit: req.Limit, + Notifications: instanceQuotaNotificationsPbToCommand(req.Notifications), + } +} + +func instanceQuotaUnitPbToCommand(unit quota.Unit) command.QuotaUnit { + switch unit { + case quota.Unit_UNIT_REQUESTS_ALL_AUTHENTICATED: + return command.QuotaRequestsAllAuthenticated + case quota.Unit_UNIT_ACTIONS_ALL_RUN_SECONDS: + return command.QuotaActionsAllRunsSeconds + case quota.Unit_UNIT_UNIMPLEMENTED: + fallthrough + default: + return command.QuotaUnit(unit.String()) + } +} + +func instanceQuotaNotificationsPbToCommand(req []*quota.Notification) command.QuotaNotifications { + notifications := make([]*command.QuotaNotification, len(req)) + for idx, item := range req { + notifications[idx] = &command.QuotaNotification{ + Percent: uint16(item.Percent), + Repeat: item.Repeat, + CallURL: item.CallUrl, + } + } + return notifications +} diff --git a/internal/api/http/cookie.go b/internal/api/http/cookie.go index 68eb50b3b..c7867646c 100644 --- a/internal/api/http/cookie.go +++ b/internal/api/http/cookie.go @@ -72,7 +72,9 @@ func WithPath(path string) CookieHandlerOpt { func WithMaxAge(maxAge int) CookieHandlerOpt { return func(c *CookieHandler) { c.maxAge = maxAge - c.securecookie.MaxAge(maxAge) + if c.securecookie != nil { + c.securecookie.MaxAge(maxAge) + } } } diff --git a/internal/api/http/middleware/access_interceptor.go b/internal/api/http/middleware/access_interceptor.go new file mode 100644 index 000000000..0de0e8c19 --- /dev/null +++ b/internal/api/http/middleware/access_interceptor.go @@ -0,0 +1,102 @@ +package middleware + +import ( + "math" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/api/authz" + http_utils "github.com/zitadel/zitadel/internal/api/http" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/logstore/emitters/access" + "github.com/zitadel/zitadel/internal/telemetry/tracing" +) + +type AccessInterceptor struct { + svc *logstore.Service + cookieHandler *http_utils.CookieHandler + limitConfig *AccessConfig +} + +type AccessConfig struct { + ExhaustedCookieKey string + ExhaustedCookieMaxAge time.Duration +} + +func NewAccessInterceptor(svc *logstore.Service, cookieConfig *AccessConfig) *AccessInterceptor { + return &AccessInterceptor{ + svc: svc, + cookieHandler: http_utils.NewCookieHandler( + http_utils.WithUnsecure(), + http_utils.WithMaxAge(int(math.Floor(cookieConfig.ExhaustedCookieMaxAge.Seconds()))), + ), + limitConfig: cookieConfig, + } +} + +func (a *AccessInterceptor) Handle(next http.Handler) http.Handler { + if !a.svc.Enabled() { + return next + } + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + + ctx := request.Context() + var err error + + tracingCtx, span := tracing.NewServerInterceptorSpan(ctx) + defer func() { span.EndWithError(err) }() + + wrappedWriter := &statusRecorder{ResponseWriter: writer, status: 0} + + instance := authz.GetInstance(ctx) + remaining := a.svc.Limit(tracingCtx, instance.InstanceID()) + limit := remaining != nil && *remaining == 0 + + a.cookieHandler.SetCookie(wrappedWriter, a.limitConfig.ExhaustedCookieKey, request.Host, strconv.FormatBool(limit)) + + if limit { + wrappedWriter.WriteHeader(http.StatusTooManyRequests) + wrappedWriter.ignoreWrites = true + } + + next.ServeHTTP(wrappedWriter, request) + + requestURL := request.RequestURI + unescapedURL, err := url.QueryUnescape(requestURL) + if err != nil { + logging.WithError(err).WithField("url", requestURL).Warning("failed to unescape request url") + // err = nil is effective because of deferred tracing span end + err = nil + } + a.svc.Handle(tracingCtx, &access.Record{ + LogDate: time.Now(), + Protocol: access.HTTP, + RequestURL: unescapedURL, + ResponseStatus: uint32(wrappedWriter.status), + RequestHeaders: request.Header, + ResponseHeaders: writer.Header(), + InstanceID: instance.InstanceID(), + ProjectID: instance.ProjectID(), + RequestedDomain: instance.RequestedDomain(), + RequestedHost: instance.RequestedHost(), + }) + }) +} + +type statusRecorder struct { + http.ResponseWriter + status int + ignoreWrites bool +} + +func (r *statusRecorder) WriteHeader(status int) { + if r.ignoreWrites { + return + } + r.status = status + r.ResponseWriter.WriteHeader(status) +} diff --git a/internal/api/http/middleware/instance_interceptor_test.go b/internal/api/http/middleware/instance_interceptor_test.go index 40e091f67..639cede84 100644 --- a/internal/api/http/middleware/instance_interceptor_test.go +++ b/internal/api/http/middleware/instance_interceptor_test.go @@ -244,6 +244,10 @@ func (m *mockInstanceVerifier) InstanceByHost(_ context.Context, host string) (a return &mockInstance{}, nil } +func (m *mockInstanceVerifier) InstanceByID(context.Context) (authz.Instance, error) { + return nil, nil +} + type mockInstance struct{} func (m *mockInstance) InstanceID() string { diff --git a/internal/api/oidc/client.go b/internal/api/oidc/client.go index 2654e6832..80fa5b61d 100644 --- a/internal/api/oidc/client.go +++ b/internal/api/oidc/client.go @@ -427,7 +427,7 @@ func (o *OPStorage) userinfoFlows(ctx context.Context, resourceOwner string, use apiFields, action.Script, action.Name, - append(actions.ActionToOptions(action), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(action), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { @@ -583,7 +583,7 @@ func (o *OPStorage) privateClaimsFlows(ctx context.Context, userID string, claim apiFields, action.Script, action.Name, - append(actions.ActionToOptions(action), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(action), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { diff --git a/internal/api/oidc/op.go b/internal/api/oidc/op.go index a3ab352a2..2ef3b2932 100644 --- a/internal/api/oidc/op.go +++ b/internal/api/oidc/op.go @@ -73,13 +73,13 @@ type OPStorage struct { assetAPIPrefix func(ctx context.Context) string } -func NewProvider(ctx context.Context, config Config, defaultLogoutRedirectURI string, externalSecure bool, command *command.Commands, query *query.Queries, repo repository.Repository, encryptionAlg crypto.EncryptionAlgorithm, cryptoKey []byte, es *eventstore.Eventstore, projections *sql.DB, userAgentCookie, instanceHandler func(http.Handler) http.Handler) (op.OpenIDProvider, error) { +func NewProvider(ctx context.Context, config Config, defaultLogoutRedirectURI string, externalSecure bool, command *command.Commands, query *query.Queries, repo repository.Repository, encryptionAlg crypto.EncryptionAlgorithm, cryptoKey []byte, es *eventstore.Eventstore, projections *sql.DB, userAgentCookie, instanceHandler, accessHandler func(http.Handler) http.Handler) (op.OpenIDProvider, error) { opConfig, err := createOPConfig(config, defaultLogoutRedirectURI, cryptoKey) if err != nil { return nil, caos_errs.ThrowInternal(err, "OIDC-EGrqd", "cannot create op config: %w") } storage := newStorage(config, command, query, repo, encryptionAlg, es, projections, externalSecure) - options, err := createOptions(config, externalSecure, userAgentCookie, instanceHandler) + options, err := createOptions(config, externalSecure, userAgentCookie, instanceHandler, accessHandler) if err != nil { return nil, caos_errs.ThrowInternal(err, "OIDC-D3gq1", "cannot create options: %w") } @@ -117,7 +117,7 @@ func createOPConfig(config Config, defaultLogoutRedirectURI string, cryptoKey [] return opConfig, nil } -func createOptions(config Config, externalSecure bool, userAgentCookie, instanceHandler func(http.Handler) http.Handler) ([]op.Option, error) { +func createOptions(config Config, externalSecure bool, userAgentCookie, instanceHandler, accessHandler func(http.Handler) http.Handler) ([]op.Option, error) { metricTypes := []metrics.MetricType{metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode, metrics.MetricTypeTotalCount} options := []op.Option{ op.WithHttpInterceptors( @@ -127,6 +127,7 @@ func createOptions(config Config, externalSecure bool, userAgentCookie, instance instanceHandler, userAgentCookie, http_utils.CopyHeadersToContext, + accessHandler, ), } if !externalSecure { diff --git a/internal/api/saml/provider.go b/internal/api/saml/provider.go index aec8f0e8f..eaab04d68 100644 --- a/internal/api/saml/provider.go +++ b/internal/api/saml/provider.go @@ -40,7 +40,8 @@ func NewProvider( es *eventstore.Eventstore, projections *sql.DB, instanceHandler, - userAgentCookie func(http.Handler) http.Handler, + userAgentCookie, + accessHandler func(http.Handler) http.Handler, ) (*provider.Provider, error) { metricTypes := []metrics.MetricType{metrics.MetricTypeRequestCount, metrics.MetricTypeStatusCode, metrics.MetricTypeTotalCount} @@ -64,6 +65,7 @@ func NewProvider( middleware.NoCacheInterceptor().Handler, instanceHandler, userAgentCookie, + accessHandler, http_utils.CopyHeadersToContext, ), } diff --git a/internal/api/ui/console/console.go b/internal/api/ui/console/console.go index 461dfffe7..87b68532d 100644 --- a/internal/api/ui/console/console.go +++ b/internal/api/ui/console/console.go @@ -88,7 +88,7 @@ func (f *file) Stat() (_ fs.FileInfo, err error) { return f, nil } -func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, instanceHandler func(http.Handler) http.Handler, customerPortal string) (http.Handler, error) { +func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, instanceHandler, accessInterceptor func(http.Handler) http.Handler, customerPortal string) (http.Handler, error) { fSys, err := fs.Sub(static, "static") if err != nil { return nil, err @@ -103,7 +103,7 @@ func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, inst handler := mux.NewRouter() - handler.Use(instanceHandler, security) + handler.Use(instanceHandler, security, accessInterceptor) handler.Handle(envRequestPath, middleware.TelemetryHandler()(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { url := http_util.BuildOrigin(r.Host, externalSecure) environmentJSON, err := createEnvironmentJSON(url, issuer(r), authz.GetInstance(r.Context()).ConsoleClientID(), customerPortal) diff --git a/internal/api/ui/login/custom_action.go b/internal/api/ui/login/custom_action.go index 50baa9cfd..c82588e18 100644 --- a/internal/api/ui/login/custom_action.go +++ b/internal/api/ui/login/custom_action.go @@ -106,7 +106,7 @@ func (l *Login) runPostExternalAuthenticationActions( apiFields, a.Script, a.Name, - append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { @@ -175,7 +175,7 @@ func (l *Login) runPostInternalAuthenticationActions( apiFields, a.Script, a.Name, - append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { @@ -274,7 +274,7 @@ func (l *Login) runPreCreationActions( apiFields, a.Script, a.Name, - append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { @@ -332,7 +332,7 @@ func (l *Login) runPostCreationActions( apiFields, a.Script, a.Name, - append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx), actions.WithLogger(actions.ServerLog))..., + append(actions.ActionToOptions(a), actions.WithHTTP(actionCtx))..., ) cancel() if err != nil { diff --git a/internal/api/ui/login/login.go b/internal/api/ui/login/login.go index f28c0072c..85e9380a6 100644 --- a/internal/api/ui/login/login.go +++ b/internal/api/ui/login/login.go @@ -69,6 +69,7 @@ func CreateLogin(config Config, oidcInstanceHandler, samlInstanceHandler mux.MiddlewareFunc, assetCache mux.MiddlewareFunc, + accessHandler mux.MiddlewareFunc, userCodeAlg crypto.EncryptionAlgorithm, idpConfigAlg crypto.EncryptionAlgorithm, csrfCookieKey []byte, @@ -94,7 +95,7 @@ func CreateLogin(config Config, cacheInterceptor := createCacheInterceptor(config.Cache.MaxAge, config.Cache.SharedMaxAge, assetCache) security := middleware.SecurityHeaders(csp(), login.cspErrorHandler) - login.router = CreateRouter(login, statikFS, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor) + login.router = CreateRouter(login, statikFS, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor, accessHandler) login.renderer = CreateRenderer(HandlerPrefix, statikFS, staticStorage, config.LanguageCookieName) login.parser = form.NewParser() return login, nil diff --git a/internal/command/command.go b/internal/command/command.go index f68bc6efa..485bf563c 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -19,6 +19,7 @@ import ( "github.com/zitadel/zitadel/internal/repository/keypair" "github.com/zitadel/zitadel/internal/repository/org" proj_repo "github.com/zitadel/zitadel/internal/repository/project" + "github.com/zitadel/zitadel/internal/repository/quota" usr_repo "github.com/zitadel/zitadel/internal/repository/user" usr_grant_repo "github.com/zitadel/zitadel/internal/repository/usergrant" "github.com/zitadel/zitadel/internal/static" @@ -110,6 +111,7 @@ func StartCommands(es *eventstore.Eventstore, proj_repo.RegisterEventMappers(repo.eventstore) keypair.RegisterEventMappers(repo.eventstore) action.RegisterEventMappers(repo.eventstore) + quota.RegisterEventMappers(repo.eventstore) repo.userPasswordAlg = crypto.NewBCrypt(defaults.SecretGenerators.PasswordSaltCost) repo.machineKeySize = int(defaults.SecretGenerators.MachineKeySize) @@ -129,6 +131,7 @@ func StartCommands(es *eventstore.Eventstore, func AppendAndReduce(object interface { AppendEvents(...eventstore.Event) + // TODO: Why is it allowed to return an error here? Reduce() error }, events ...eventstore.Event) error { object.AppendEvents(events...) diff --git a/internal/command/instance.go b/internal/command/instance.go index 0ccee4393..8e7087c3d 100644 --- a/internal/command/instance.go +++ b/internal/command/instance.go @@ -18,6 +18,7 @@ import ( "github.com/zitadel/zitadel/internal/repository/instance" "github.com/zitadel/zitadel/internal/repository/org" "github.com/zitadel/zitadel/internal/repository/project" + "github.com/zitadel/zitadel/internal/repository/quota" "github.com/zitadel/zitadel/internal/repository/user" ) @@ -116,6 +117,9 @@ type InstanceSetup struct { RefreshTokenIdleExpiration time.Duration RefreshTokenExpiration time.Duration } + Quotas *struct { + Items []*AddQuota + } } type ZitadelConfig struct { @@ -261,6 +265,19 @@ func (c *Commands) SetUpInstance(ctx context.Context, setup *InstanceSetup) (str prepareAddDefaultEmailTemplate(instanceAgg, setup.EmailTemplate), } + if setup.Quotas != nil { + for _, q := range setup.Quotas.Items { + quotaId, err := c.idGenerator.Next() + if err != nil { + return "", "", nil, nil, err + } + + quotaAggregate := quota.NewAggregate(quotaId, instanceID, instanceID) + + validations = append(validations, c.AddQuotaCommand(quotaAggregate, q)) + } + } + for _, msg := range setup.MessageTexts { validations = append(validations, prepareSetInstanceCustomMessageTexts(instanceAgg, msg)) } diff --git a/internal/command/instancce_policy_password_age_model.go b/internal/command/instance_policy_password_age_model.go similarity index 100% rename from internal/command/instancce_policy_password_age_model.go rename to internal/command/instance_policy_password_age_model.go diff --git a/internal/command/quota.go b/internal/command/quota.go new file mode 100644 index 000000000..95e4d6fa8 --- /dev/null +++ b/internal/command/quota.go @@ -0,0 +1,207 @@ +package command + +import ( + "context" + "net/url" + "time" + + "github.com/zitadel/zitadel/internal/api/authz" + "github.com/zitadel/zitadel/internal/command/preparation" + "github.com/zitadel/zitadel/internal/domain" + "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/eventstore" + "github.com/zitadel/zitadel/internal/id" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +type QuotaUnit string + +const ( + QuotaRequestsAllAuthenticated QuotaUnit = "requests.all.authenticated" + QuotaActionsAllRunsSeconds QuotaUnit = "actions.all.runs.seconds" +) + +func (q *QuotaUnit) Enum() quota.Unit { + switch *q { + case QuotaRequestsAllAuthenticated: + return quota.RequestsAllAuthenticated + case QuotaActionsAllRunsSeconds: + return quota.ActionsAllRunsSeconds + default: + return quota.Unimplemented + } +} + +func (c *Commands) AddQuota( + ctx context.Context, + q *AddQuota, +) (*domain.ObjectDetails, error) { + instanceId := authz.GetInstance(ctx).InstanceID() + + wm, err := c.getQuotaWriteModel(ctx, instanceId, instanceId, q.Unit.Enum()) + if err != nil { + return nil, err + } + + if wm.active { + return nil, errors.ThrowAlreadyExists(nil, "COMMAND-WDfFf", "Errors.Quota.AlreadyExists") + } + + aggregateId, err := c.idGenerator.Next() + if err != nil { + return nil, err + } + aggregate := quota.NewAggregate(aggregateId, instanceId, instanceId) + + cmds, err := preparation.PrepareCommands(ctx, c.eventstore.Filter, c.AddQuotaCommand(aggregate, q)) + if err != nil { + return nil, err + } + events, err := c.eventstore.Push(ctx, cmds...) + if err != nil { + return nil, err + } + err = AppendAndReduce(wm, events...) + if err != nil { + return nil, err + } + return writeModelToObjectDetails(&wm.WriteModel), nil +} + +func (c *Commands) RemoveQuota(ctx context.Context, unit QuotaUnit) (*domain.ObjectDetails, error) { + instanceId := authz.GetInstance(ctx).InstanceID() + + wm, err := c.getQuotaWriteModel(ctx, instanceId, instanceId, unit.Enum()) + if err != nil { + return nil, err + } + + if !wm.active { + return nil, errors.ThrowNotFound(nil, "COMMAND-WDfFf", "Errors.Quota.NotFound") + } + + aggregate := quota.NewAggregate(wm.AggregateID, instanceId, instanceId) + + events := []eventstore.Command{ + quota.NewRemovedEvent(ctx, &aggregate.Aggregate, unit.Enum()), + } + pushedEvents, err := c.eventstore.Push(ctx, events...) + if err != nil { + return nil, err + } + err = AppendAndReduce(wm, pushedEvents...) + if err != nil { + return nil, err + } + return writeModelToObjectDetails(&wm.WriteModel), nil +} + +func (c *Commands) getQuotaWriteModel(ctx context.Context, instanceId, resourceOwner string, unit quota.Unit) (*quotaWriteModel, error) { + wm := newQuotaWriteModel(instanceId, resourceOwner, unit) + return wm, c.eventstore.FilterToQueryReducer(ctx, wm) +} + +type QuotaNotification struct { + Percent uint16 + Repeat bool + CallURL string +} + +type QuotaNotifications []*QuotaNotification + +func (q *QuotaNotifications) toAddedEventNotifications(idGenerator id.Generator) ([]*quota.AddedEventNotification, error) { + if q == nil { + return nil, nil + } + + notifications := make([]*quota.AddedEventNotification, len(*q)) + for idx, notification := range *q { + + id, err := idGenerator.Next() + if err != nil { + return nil, err + } + + notifications[idx] = "a.AddedEventNotification{ + ID: id, + Percent: notification.Percent, + Repeat: notification.Repeat, + CallURL: notification.CallURL, + } + } + + return notifications, nil +} + +type AddQuota struct { + Unit QuotaUnit + From time.Time + ResetInterval time.Duration + Amount uint64 + Limit bool + Notifications QuotaNotifications +} + +func (q *AddQuota) validate() error { + for _, notification := range q.Notifications { + u, err := url.Parse(notification.CallURL) + if err != nil { + return errors.ThrowInvalidArgument(err, "QUOTA-bZ0Fj", "Errors.Quota.Invalid.CallURL") + } + + if !u.IsAbs() || u.Host == "" { + return errors.ThrowInvalidArgument(nil, "QUOTA-HAYmN", "Errors.Quota.Invalid.CallURL") + } + + if notification.Percent < 1 { + return errors.ThrowInvalidArgument(nil, "QUOTA-pBfjq", "Errors.Quota.Invalid.Percent") + } + } + + if q.Unit.Enum() == quota.Unimplemented { + return errors.ThrowInvalidArgument(nil, "QUOTA-OTeSh", "Errors.Quota.Invalid.Unimplemented") + } + + if q.Amount < 1 { + return errors.ThrowInvalidArgument(nil, "QUOTA-hOKSJ", "Errors.Quota.Invalid.Amount") + } + + if q.ResetInterval < time.Minute { + return errors.ThrowInvalidArgument(nil, "QUOTA-R5otd", "Errors.Quota.Invalid.ResetInterval") + } + + if !q.Limit && len(q.Notifications) == 0 { + return errors.ThrowInvalidArgument(nil, "QUOTA-4Nv68", "Errors.Quota.Invalid.Noop") + } + + return nil +} + +func (c *Commands) AddQuotaCommand(a *quota.Aggregate, q *AddQuota) preparation.Validation { + return func() (preparation.CreateCommands, error) { + + if err := q.validate(); err != nil { + return nil, err + } + + return func(ctx context.Context, filter preparation.FilterToQueryReducer) (cmd []eventstore.Command, err error) { + + notifications, err := q.Notifications.toAddedEventNotifications(c.idGenerator) + if err != nil { + return nil, err + } + + return []eventstore.Command{quota.NewAddedEvent( + ctx, + &a.Aggregate, + q.Unit.Enum(), + q.From, + q.ResetInterval, + q.Amount, + q.Limit, + notifications, + )}, err + }, + nil + } +} diff --git a/internal/command/quota_model.go b/internal/command/quota_model.go new file mode 100644 index 000000000..123bd3555 --- /dev/null +++ b/internal/command/quota_model.go @@ -0,0 +1,54 @@ +package command + +import ( + "github.com/zitadel/zitadel/internal/eventstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +type quotaWriteModel struct { + eventstore.WriteModel + unit quota.Unit + active bool + config *quota.AddedEvent +} + +// newQuotaWriteModel aggregateId is filled by reducing unit matching events +func newQuotaWriteModel(instanceId, resourceOwner string, unit quota.Unit) *quotaWriteModel { + return "aWriteModel{ + WriteModel: eventstore.WriteModel{ + InstanceID: instanceId, + ResourceOwner: resourceOwner, + }, + unit: unit, + } +} + +func (wm *quotaWriteModel) Query() *eventstore.SearchQueryBuilder { + query := eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent). + ResourceOwner(wm.ResourceOwner). + AddQuery(). + InstanceID(wm.InstanceID). + AggregateTypes(quota.AggregateType). + EventTypes( + quota.AddedEventType, + quota.RemovedEventType, + ).EventData(map[string]interface{}{"unit": wm.unit}) + + return query.Builder() +} + +func (wm *quotaWriteModel) Reduce() error { + for _, event := range wm.Events { + switch e := event.(type) { + case *quota.AddedEvent: + wm.AggregateID = e.Aggregate().ID + wm.active = true + wm.config = e + case *quota.RemovedEvent: + wm.AggregateID = e.Aggregate().ID + wm.active = false + wm.config = nil + } + } + return wm.WriteModel.Reduce() +} diff --git a/internal/command/quota_notifications.go b/internal/command/quota_notifications.go new file mode 100644 index 000000000..15f1695df --- /dev/null +++ b/internal/command/quota_notifications.go @@ -0,0 +1,59 @@ +package command + +import ( + "context" + "math" + "time" + + "github.com/zitadel/zitadel/internal/eventstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +func (c *Commands) GetDueQuotaNotifications(ctx context.Context, config *quota.AddedEvent, periodStart time.Time, usedAbs uint64) ([]*quota.NotifiedEvent, error) { + if len(config.Notifications) == 0 { + return nil, nil + } + + aggregate := config.Aggregate() + wm, err := c.getQuotaNotificationsWriteModel(ctx, aggregate, periodStart) + if err != nil { + return nil, err + } + + usedRel := uint16(math.Floor(float64(usedAbs*100) / float64(config.Amount))) + + var dueNotifications []*quota.NotifiedEvent + for _, notification := range config.Notifications { + if notification.Percent > usedRel { + continue + } + + threshold := notification.Percent + if notification.Repeat { + threshold = uint16(math.Min(1, math.Floor(float64(usedRel)/float64(notification.Percent)))) * notification.Percent + } + + if wm.latestNotifiedThresholds[notification.ID] < threshold { + dueNotifications = append( + dueNotifications, + quota.NewNotifiedEvent( + ctx, + &aggregate, + config.Unit, + notification.ID, + notification.CallURL, + periodStart, + threshold, + usedAbs, + ), + ) + } + } + + return dueNotifications, nil +} + +func (c *Commands) getQuotaNotificationsWriteModel(ctx context.Context, aggregate eventstore.Aggregate, periodStart time.Time) (*quotaNotificationsWriteModel, error) { + wm := newQuotaNotificationsWriteModel(aggregate.ID, aggregate.InstanceID, aggregate.ResourceOwner, periodStart) + return wm, c.eventstore.FilterToQueryReducer(ctx, wm) +} diff --git a/internal/command/quota_notifications_model.go b/internal/command/quota_notifications_model.go new file mode 100644 index 000000000..41382e5bc --- /dev/null +++ b/internal/command/quota_notifications_model.go @@ -0,0 +1,45 @@ +package command + +import ( + "time" + + "github.com/zitadel/zitadel/internal/eventstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +type quotaNotificationsWriteModel struct { + eventstore.WriteModel + periodStart time.Time + latestNotifiedThresholds map[string]uint16 +} + +func newQuotaNotificationsWriteModel(aggregateId, instanceId, resourceOwner string, periodStart time.Time) *quotaNotificationsWriteModel { + return "aNotificationsWriteModel{ + WriteModel: eventstore.WriteModel{ + AggregateID: aggregateId, + InstanceID: instanceId, + ResourceOwner: resourceOwner, + }, + periodStart: periodStart, + latestNotifiedThresholds: make(map[string]uint16), + } +} + +func (wm *quotaNotificationsWriteModel) Query() *eventstore.SearchQueryBuilder { + return eventstore.NewSearchQueryBuilder(eventstore.ColumnsEvent). + ResourceOwner(wm.ResourceOwner). + AddQuery(). + InstanceID(wm.InstanceID). + AggregateTypes(quota.AggregateType). + AggregateIDs(wm.AggregateID). + CreationDateAfter(wm.periodStart). + EventTypes(quota.NotifiedEventType).Builder() +} + +func (wm *quotaNotificationsWriteModel) Reduce() error { + for _, event := range wm.Events { + e := event.(*quota.NotifiedEvent) + wm.latestNotifiedThresholds[e.ID] = e.Threshold + } + return wm.WriteModel.Reduce() +} diff --git a/internal/command/quota_period.go b/internal/command/quota_period.go new file mode 100644 index 000000000..c82d1707e --- /dev/null +++ b/internal/command/quota_period.go @@ -0,0 +1,25 @@ +package command + +import ( + "context" + "time" + + "github.com/zitadel/zitadel/internal/repository/quota" +) + +func (c *Commands) GetCurrentQuotaPeriod(ctx context.Context, instanceID string, unit quota.Unit) (*quota.AddedEvent, time.Time, error) { + wm, err := c.getQuotaWriteModel(ctx, instanceID, instanceID, unit) + if err != nil || !wm.active { + return nil, time.Time{}, err + } + + return wm.config, pushPeriodStart(wm.config.From, wm.config.ResetInterval, time.Now()), nil +} + +func pushPeriodStart(from time.Time, interval time.Duration, now time.Time) time.Time { + next := from.Add(interval) + if next.After(now) { + return from + } + return pushPeriodStart(next, interval, now) +} diff --git a/internal/command/quota_report.go b/internal/command/quota_report.go new file mode 100644 index 000000000..5228a7a4d --- /dev/null +++ b/internal/command/quota_report.go @@ -0,0 +1,58 @@ +package command + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/zitadel/zitadel/internal/repository/quota" +) + +// ReportUsage calls notification hooks and emits the notified events +func (c *Commands) ReportUsage(ctx context.Context, dueNotifications []*quota.NotifiedEvent) error { + for _, notification := range dueNotifications { + + if err := notify(ctx, notification); err != nil { + if err != nil { + return err + } + } + + if _, err := c.eventstore.Push(ctx, notification); err != nil { + return err + } + } + + return nil +} + +func notify(ctx context.Context, notification *quota.NotifiedEvent) error { + payload, err := json.Marshal(notification) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, notification.CallURL, bytes.NewReader(payload)) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if err = resp.Body.Close(); err != nil { + return err + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("calling url %s returned %s", notification.CallURL, resp.Status) + } + + return nil +} diff --git a/internal/errors/resource_exhausted.go b/internal/errors/resource_exhausted.go new file mode 100644 index 000000000..1485fa361 --- /dev/null +++ b/internal/errors/resource_exhausted.go @@ -0,0 +1,48 @@ +package errors + +import ( + "fmt" +) + +var ( + _ ResourceExhausted = (*ResourceExhaustedError)(nil) + _ Error = (*ResourceExhaustedError)(nil) +) + +type ResourceExhausted interface { + error + IsResourceExhausted() +} + +type ResourceExhaustedError struct { + *CaosError +} + +func ThrowResourceExhausted(parent error, id, message string) error { + return &ResourceExhaustedError{CreateCaosError(parent, id, message)} +} + +func ThrowResourceExhaustedf(parent error, id, format string, a ...interface{}) error { + return ThrowResourceExhausted(parent, id, fmt.Sprintf(format, a...)) +} + +func (err *ResourceExhaustedError) IsResourceExhausted() {} + +func IsResourceExhausted(err error) bool { + //nolint:errorlint + _, ok := err.(ResourceExhausted) + return ok +} + +func (err *ResourceExhaustedError) Is(target error) bool { + //nolint:errorlint + t, ok := target.(*ResourceExhaustedError) + if !ok { + return false + } + return err.CaosError.Is(t.CaosError) +} + +func (err *ResourceExhaustedError) Unwrap() error { + return err.CaosError +} diff --git a/internal/errors/resource_exhausted_test.go b/internal/errors/resource_exhausted_test.go new file mode 100644 index 000000000..dd2bd7866 --- /dev/null +++ b/internal/errors/resource_exhausted_test.go @@ -0,0 +1,34 @@ +package errors_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + + caos_errs "github.com/zitadel/zitadel/internal/errors" +) + +func TestResourceExhaustedError(t *testing.T) { + var err interface{} = new(caos_errs.ResourceExhaustedError) + _, ok := err.(caos_errs.ResourceExhausted) + assert.True(t, ok) +} + +func TestThrowResourceExhaustedf(t *testing.T) { + err := caos_errs.ThrowResourceExhaustedf(nil, "id", "msg") + // TODO: refactor errors package + //nolint:errorlint + _, ok := err.(*caos_errs.ResourceExhaustedError) + assert.True(t, ok) +} + +func TestIsResourceExhausted(t *testing.T) { + err := caos_errs.ThrowResourceExhausted(nil, "id", "msg") + ok := caos_errs.IsResourceExhausted(err) + assert.True(t, ok) + + err = errors.New("I am found!") + ok = caos_errs.IsResourceExhausted(err) + assert.False(t, ok) +} diff --git a/internal/eventstore/handler/crdb/init.go b/internal/eventstore/handler/crdb/init.go index 2039fcd97..9ca9c34a9 100644 --- a/internal/eventstore/handler/crdb/init.go +++ b/internal/eventstore/handler/crdb/init.go @@ -119,6 +119,7 @@ const ( ColumnTypeJSONB ColumnTypeBytes ColumnTypeTimestamp + ColumnTypeInterval ColumnTypeEnum ColumnTypeEnumArray ColumnTypeInt64 @@ -389,6 +390,8 @@ func columnType(columnType ColumnType) string { return "TEXT[]" case ColumnTypeTimestamp: return "TIMESTAMPTZ" + case ColumnTypeInterval: + return "INTERVAL" case ColumnTypeEnum: return "SMALLINT" case ColumnTypeEnumArray: diff --git a/internal/logstore/config.go b/internal/logstore/config.go new file mode 100644 index 000000000..3e8a8d327 --- /dev/null +++ b/internal/logstore/config.go @@ -0,0 +1,11 @@ +package logstore + +type Configs struct { + Access *Config + Execution *Config +} + +type Config struct { + Database *EmitterConfig + Stdout *EmitterConfig +} diff --git a/internal/logstore/debouncer.go b/internal/logstore/debouncer.go new file mode 100644 index 000000000..e299c08a8 --- /dev/null +++ b/internal/logstore/debouncer.go @@ -0,0 +1,92 @@ +package logstore + +import ( + "context" + "sync" + "time" + + "github.com/benbjohnson/clock" + "github.com/zitadel/logging" +) + +type bulkSink interface { + sendBulk(ctx context.Context, bulk []LogRecord) error +} + +var _ bulkSink = bulkSinkFunc(nil) + +type bulkSinkFunc func(ctx context.Context, items []LogRecord) error + +func (s bulkSinkFunc) sendBulk(ctx context.Context, items []LogRecord) error { + return s(ctx, items) +} + +type debouncer struct { + // Storing context.Context in a struct is generally bad practice + // https://go.dev/blog/context-and-structs + // However, debouncer starts a go routine that triggers side effects itself. + // So, there is no incoming context.Context available when these events trigger. + // The only context we can use for the side effects is the app context. + // Because this can be cancelled by os signals, it's the better solution than creating new background contexts. + binarySignaledCtx context.Context + clock clock.Clock + ticker *clock.Ticker + mux sync.Mutex + cfg DebouncerConfig + storage bulkSink + cache []LogRecord + cacheLen uint +} + +type DebouncerConfig struct { + MinFrequency time.Duration + MaxBulkSize uint +} + +func newDebouncer(binarySignaledCtx context.Context, cfg DebouncerConfig, clock clock.Clock, ship bulkSink) *debouncer { + a := &debouncer{ + binarySignaledCtx: binarySignaledCtx, + clock: clock, + cfg: cfg, + storage: ship, + } + + if cfg.MinFrequency > 0 { + a.ticker = clock.Ticker(cfg.MinFrequency) + go a.shipOnTicks() + } + return a +} + +func (d *debouncer) add(item LogRecord) { + d.mux.Lock() + defer d.mux.Unlock() + d.cache = append(d.cache, item) + d.cacheLen++ + if d.cfg.MaxBulkSize > 0 && d.cacheLen >= d.cfg.MaxBulkSize { + // Add should not block and release the lock + go d.ship() + } +} + +func (d *debouncer) ship() { + if d.cacheLen == 0 { + return + } + d.mux.Lock() + defer d.mux.Unlock() + if err := d.storage.sendBulk(d.binarySignaledCtx, d.cache); err != nil { + logging.WithError(err).WithField("size", len(d.cache)).Error("storing bulk failed") + } + d.cache = nil + d.cacheLen = 0 + if d.cfg.MinFrequency > 0 { + d.ticker.Reset(d.cfg.MinFrequency) + } +} + +func (d *debouncer) shipOnTicks() { + for range d.ticker.C { + d.ship() + } +} diff --git a/internal/logstore/emitter.go b/internal/logstore/emitter.go new file mode 100644 index 000000000..a7a5151af --- /dev/null +++ b/internal/logstore/emitter.go @@ -0,0 +1,112 @@ +package logstore + +import ( + "context" + "fmt" + "time" + + "github.com/benbjohnson/clock" + "github.com/zitadel/logging" +) + +type EmitterConfig struct { + Enabled bool + Keep time.Duration + CleanupInterval time.Duration + Debounce *DebouncerConfig +} + +type emitter struct { + enabled bool + ctx context.Context + debouncer *debouncer + emitter LogEmitter + clock clock.Clock +} + +type LogRecord interface { + Normalize() LogRecord +} + +type LogRecordFunc func() LogRecord + +func (r LogRecordFunc) Normalize() LogRecord { + return r() +} + +type LogEmitter interface { + Emit(ctx context.Context, bulk []LogRecord) error +} + +type LogEmitterFunc func(ctx context.Context, bulk []LogRecord) error + +func (l LogEmitterFunc) Emit(ctx context.Context, bulk []LogRecord) error { + return l(ctx, bulk) +} + +type LogCleanupper interface { + LogEmitter + Cleanup(ctx context.Context, keep time.Duration) error +} + +// NewEmitter accepts Clock from github.com/benbjohnson/clock so we can control timers and tickers in the unit tests +func NewEmitter(ctx context.Context, clock clock.Clock, cfg *EmitterConfig, logger LogEmitter) (*emitter, error) { + svc := &emitter{ + enabled: cfg != nil && cfg.Enabled, + ctx: ctx, + emitter: logger, + clock: clock, + } + + if !svc.enabled { + return svc, nil + } + + if cfg.Debounce != nil && (cfg.Debounce.MinFrequency > 0 || cfg.Debounce.MaxBulkSize > 0) { + svc.debouncer = newDebouncer(ctx, *cfg.Debounce, clock, newStorageBulkSink(svc.emitter)) + } + + cleanupper, ok := logger.(LogCleanupper) + if !ok { + if cfg.Keep != 0 { + return nil, fmt.Errorf("cleaning up for this storage type is not supported, so keep duration must be 0, but is %d", cfg.Keep) + } + if cfg.CleanupInterval != 0 { + return nil, fmt.Errorf("cleaning up for this storage type is not supported, so cleanup interval duration must be 0, but is %d", cfg.Keep) + } + + return svc, nil + } + + if cfg.Keep != 0 && cfg.CleanupInterval != 0 { + go svc.startCleanupping(cleanupper, cfg.CleanupInterval, cfg.Keep) + } + return svc, nil +} + +func (s *emitter) startCleanupping(cleanupper LogCleanupper, cleanupInterval, keep time.Duration) { + for range s.clock.Tick(cleanupInterval) { + if err := cleanupper.Cleanup(s.ctx, keep); err != nil { + logging.WithError(err).Error("cleaning up logs failed") + } + } +} + +func (s *emitter) Emit(ctx context.Context, record LogRecord) (err error) { + if !s.enabled { + return nil + } + + if s.debouncer != nil { + s.debouncer.add(record) + return nil + } + + return s.emitter.Emit(ctx, []LogRecord{record}) +} + +func newStorageBulkSink(emitter LogEmitter) bulkSinkFunc { + return func(ctx context.Context, bulk []LogRecord) error { + return emitter.Emit(ctx, bulk) + } +} diff --git a/internal/logstore/emitters/access/database.go b/internal/logstore/emitters/access/database.go new file mode 100644 index 000000000..96d60f596 --- /dev/null +++ b/internal/logstore/emitters/access/database.go @@ -0,0 +1,159 @@ +package access + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "strings" + "time" + + "github.com/Masterminds/squirrel" + "github.com/zitadel/logging" + "google.golang.org/grpc/codes" + + zitadel_http "github.com/zitadel/zitadel/internal/api/http" + caos_errors "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +const ( + accessLogsTable = "logstore.access" + accessTimestampCol = "log_date" + accessProtocolCol = "protocol" + accessRequestURLCol = "request_url" + accessResponseStatusCol = "response_status" + accessRequestHeadersCol = "request_headers" + accessResponseHeadersCol = "response_headers" + accessInstanceIdCol = "instance_id" + accessProjectIdCol = "project_id" + accessRequestedDomainCol = "requested_domain" + accessRequestedHostCol = "requested_host" +) + +var _ logstore.UsageQuerier = (*databaseLogStorage)(nil) +var _ logstore.LogCleanupper = (*databaseLogStorage)(nil) + +type databaseLogStorage struct { + dbClient *sql.DB +} + +func NewDatabaseLogStorage(dbClient *sql.DB) *databaseLogStorage { + return &databaseLogStorage{dbClient: dbClient} +} + +func (l *databaseLogStorage) QuotaUnit() quota.Unit { + return quota.RequestsAllAuthenticated +} + +func (l *databaseLogStorage) Emit(ctx context.Context, bulk []logstore.LogRecord) error { + builder := squirrel.Insert(accessLogsTable). + Columns( + accessTimestampCol, + accessProtocolCol, + accessRequestURLCol, + accessResponseStatusCol, + accessRequestHeadersCol, + accessResponseHeadersCol, + accessInstanceIdCol, + accessProjectIdCol, + accessRequestedDomainCol, + accessRequestedHostCol, + ). + PlaceholderFormat(squirrel.Dollar) + + for idx := range bulk { + item := bulk[idx].(*Record) + builder = builder.Values( + item.LogDate, + item.Protocol, + item.RequestURL, + item.ResponseStatus, + item.RequestHeaders, + item.ResponseHeaders, + item.InstanceID, + item.ProjectID, + item.RequestedDomain, + item.RequestedHost, + ) + } + + stmt, args, err := builder.ToSql() + if err != nil { + return caos_errors.ThrowInternal(err, "ACCESS-KOS7I", "Errors.Internal") + } + + result, err := l.dbClient.ExecContext(ctx, stmt, args...) + if err != nil { + return caos_errors.ThrowInternal(err, "ACCESS-alnT9", "Errors.Access.StorageFailed") + } + + rows, err := result.RowsAffected() + if err != nil { + return caos_errors.ThrowInternal(err, "ACCESS-7KIpL", "Errors.Internal") + } + + logging.WithFields("rows", rows).Debug("successfully stored access logs") + return nil +} + +// TODO: AS OF SYSTEM TIME +func (l *databaseLogStorage) QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) { + stmt, args, err := squirrel.Select( + fmt.Sprintf("count(%s)", accessInstanceIdCol), + ). + From(accessLogsTable). + Where(squirrel.And{ + squirrel.Eq{accessInstanceIdCol: instanceId}, + squirrel.GtOrEq{accessTimestampCol: start}, + squirrel.Expr(fmt.Sprintf(`%s #>> '{%s,0}' = '[REDACTED]'`, accessRequestHeadersCol, strings.ToLower(zitadel_http.Authorization))), + squirrel.NotLike{accessRequestURLCol: "%/zitadel.system.v1.SystemService/%"}, + squirrel.NotLike{accessRequestURLCol: "%/system/v1/%"}, + squirrel.Or{ + squirrel.And{ + squirrel.Eq{accessProtocolCol: HTTP}, + squirrel.NotEq{accessResponseStatusCol: http.StatusForbidden}, + squirrel.NotEq{accessResponseStatusCol: http.StatusInternalServerError}, + squirrel.NotEq{accessResponseStatusCol: http.StatusTooManyRequests}, + }, + squirrel.And{ + squirrel.Eq{accessProtocolCol: GRPC}, + squirrel.NotEq{accessResponseStatusCol: codes.PermissionDenied}, + squirrel.NotEq{accessResponseStatusCol: codes.Internal}, + squirrel.NotEq{accessResponseStatusCol: codes.ResourceExhausted}, + }, + }, + }). + PlaceholderFormat(squirrel.Dollar). + ToSql() + + if err != nil { + return 0, caos_errors.ThrowInternal(err, "ACCESS-V9Sde", "Errors.Internal") + } + + var count uint64 + if err = l.dbClient. + QueryRowContext(ctx, stmt, args...). + Scan(&count); err != nil { + return 0, caos_errors.ThrowInternal(err, "ACCESS-pBPrM", "Errors.Logstore.Access.ScanFailed") + } + + return count, nil +} + +func (l *databaseLogStorage) Cleanup(ctx context.Context, keep time.Duration) error { + stmt, args, err := squirrel.Delete(accessLogsTable). + Where(squirrel.LtOrEq{accessTimestampCol: time.Now().Add(-keep)}). + PlaceholderFormat(squirrel.Dollar). + ToSql() + + if err != nil { + return caos_errors.ThrowInternal(err, "ACCESS-2oTh6", "Errors.Internal") + } + + execCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = l.dbClient.ExecContext(execCtx, stmt, args...) + return err +} diff --git a/internal/logstore/emitters/access/record.go b/internal/logstore/emitters/access/record.go new file mode 100644 index 000000000..17d1a7506 --- /dev/null +++ b/internal/logstore/emitters/access/record.go @@ -0,0 +1,91 @@ +package access + +import ( + "strings" + "time" + + zitadel_http "github.com/zitadel/zitadel/internal/api/http" + "github.com/zitadel/zitadel/internal/logstore" +) + +var _ logstore.LogRecord = (*Record)(nil) + +type Record struct { + LogDate time.Time `json:"logDate"` + Protocol Protocol `json:"protocol"` + RequestURL string `json:"requestUrl"` + ResponseStatus uint32 `json:"responseStatus"` + // RequestHeaders are plain maps so varying implementation + // between HTTP and gRPC don't interfere with each other + RequestHeaders map[string][]string `json:"requestHeaders"` + // ResponseHeaders are plain maps so varying implementation + // between HTTP and gRPC don't interfere with each other + ResponseHeaders map[string][]string `json:"responseHeaders"` + InstanceID string `json:"instanceId"` + ProjectID string `json:"projectId"` + RequestedDomain string `json:"requestedDomain"` + RequestedHost string `json:"requestedHost"` +} + +type Protocol uint8 + +const ( + GRPC Protocol = iota + HTTP + + redacted = "[REDACTED]" +) + +func (a Record) Normalize() logstore.LogRecord { + a.RequestedDomain = cutString(a.RequestedDomain, 200) + a.RequestURL = cutString(a.RequestURL, 200) + normalizeHeaders(a.RequestHeaders, strings.ToLower(zitadel_http.Authorization), "grpcgateway-authorization", "cookie", "grpcgateway-cookie") + normalizeHeaders(a.ResponseHeaders, "set-cookie") + return &a +} + +const maxValuesPerKey = 10 + +// normalizeHeaders lowers all header keys and redacts secrets +func normalizeHeaders(header map[string][]string, redactKeysLower ...string) { + lowerKeys(header) + redactKeys(header, redactKeysLower...) + pruneKeys(header) +} + +func lowerKeys(header map[string][]string) { + for k, v := range header { + delete(header, k) + header[strings.ToLower(k)] = v + } +} + +func redactKeys(header map[string][]string, redactKeysLower ...string) { + for _, redactKey := range redactKeysLower { + if _, ok := header[redactKey]; ok { + header[redactKey] = []string{redacted} + } + } +} + +func pruneKeys(header map[string][]string) { + for key, value := range header { + valueItems := make([]string, 0, maxValuesPerKey) + for i, valueItem := range value { + // Max 10 header values per key + if i > maxValuesPerKey { + break + } + // Max 200 value length + valueItems = append(valueItems, cutString(valueItem, 200)) + } + header[key] = valueItems + } +} + +func cutString(str string, pos int) string { + if len(str) <= pos { + return str + } + return str[:pos-1] +} diff --git a/internal/logstore/emitters/execution/database.go b/internal/logstore/emitters/execution/database.go new file mode 100644 index 000000000..1c64829eb --- /dev/null +++ b/internal/logstore/emitters/execution/database.go @@ -0,0 +1,135 @@ +package execution + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/Masterminds/squirrel" + "github.com/zitadel/logging" + + caos_errors "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +const ( + executionLogsTable = "logstore.execution" + executionTimestampCol = "log_date" + executionTookCol = "took" + executionMessageCol = "message" + executionLogLevelCol = "loglevel" + executionInstanceIdCol = "instance_id" + executionActionIdCol = "action_id" + executionMetadataCol = "metadata" +) + +var _ logstore.UsageQuerier = (*databaseLogStorage)(nil) +var _ logstore.LogCleanupper = (*databaseLogStorage)(nil) + +type databaseLogStorage struct { + dbClient *sql.DB +} + +func NewDatabaseLogStorage(dbClient *sql.DB) *databaseLogStorage { + return &databaseLogStorage{dbClient: dbClient} +} + +func (l *databaseLogStorage) QuotaUnit() quota.Unit { + return quota.ActionsAllRunsSeconds +} + +func (l *databaseLogStorage) Emit(ctx context.Context, bulk []logstore.LogRecord) error { + builder := squirrel.Insert(executionLogsTable). + Columns( + executionTimestampCol, + executionTookCol, + executionMessageCol, + executionLogLevelCol, + executionInstanceIdCol, + executionActionIdCol, + executionMetadataCol, + ). + PlaceholderFormat(squirrel.Dollar) + + for idx := range bulk { + item := bulk[idx].(*Record) + + var took interface{} + if item.Took > 0 { + took = item.Took + } + + builder = builder.Values( + item.LogDate, + took, + item.Message, + item.LogLevel, + item.InstanceID, + item.ActionID, + item.Metadata, + ) + } + + stmt, args, err := builder.ToSql() + if err != nil { + return caos_errors.ThrowInternal(err, "EXEC-KOS7I", "Errors.Internal") + } + + result, err := l.dbClient.ExecContext(ctx, stmt, args...) + if err != nil { + return caos_errors.ThrowInternal(err, "EXEC-0j6i5", "Errors.Access.StorageFailed") + } + + rows, err := result.RowsAffected() + if err != nil { + return caos_errors.ThrowInternal(err, "EXEC-MGchJ", "Errors.Internal") + } + + logging.WithFields("rows", rows).Debug("successfully stored execution logs") + return nil +} + +// TODO: AS OF SYSTEM TIME +func (l *databaseLogStorage) QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) { + stmt, args, err := squirrel.Select( + fmt.Sprintf("COALESCE(SUM(%s)::INT,0)", executionTookCol), + ). + From(executionLogsTable). + Where(squirrel.And{ + squirrel.Eq{executionInstanceIdCol: instanceId}, + squirrel.GtOrEq{executionTimestampCol: start}, + squirrel.NotEq{executionTookCol: nil}, + }). + PlaceholderFormat(squirrel.Dollar). + ToSql() + + if err != nil { + return 0, caos_errors.ThrowInternal(err, "EXEC-DXtzg", "Errors.Internal") + } + + var durationSeconds uint64 + if err = l.dbClient. + QueryRowContext(ctx, stmt, args...). + Scan(&durationSeconds); err != nil { + return 0, caos_errors.ThrowInternal(err, "EXEC-Ad8nP", "Errors.Logstore.Execution.ScanFailed") + } + return durationSeconds, nil +} + +func (l *databaseLogStorage) Cleanup(ctx context.Context, keep time.Duration) error { + stmt, args, err := squirrel.Delete(executionLogsTable). + Where(squirrel.LtOrEq{executionTimestampCol: time.Now().Add(-keep)}). + PlaceholderFormat(squirrel.Dollar). + ToSql() + + if err != nil { + return caos_errors.ThrowInternal(err, "EXEC-Bja8V", "Errors.Internal") + } + + execCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + _, err = l.dbClient.ExecContext(execCtx, stmt, args...) + return err +} diff --git a/internal/logstore/emitters/execution/record.go b/internal/logstore/emitters/execution/record.go new file mode 100644 index 000000000..c45790622 --- /dev/null +++ b/internal/logstore/emitters/execution/record.go @@ -0,0 +1,33 @@ +package execution + +import ( + "time" + + "github.com/sirupsen/logrus" + + "github.com/zitadel/zitadel/internal/logstore" +) + +var _ logstore.LogRecord = (*Record)(nil) + +type Record struct { + LogDate time.Time `json:"logDate"` + Took time.Duration `json:"took"` + Message string `json:"message"` + LogLevel logrus.Level `json:"logLevel"` + InstanceID string `json:"instanceId"` + ActionID string `json:"actionId,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` +} + +func (e Record) Normalize() logstore.LogRecord { + e.Message = cutString(e.Message, 2000) + return &e +} + +func cutString(str string, pos int) string { + if len(str) <= pos { + return str + } + return str[:pos] +} diff --git a/internal/logstore/emitters/mock/inmem.go b/internal/logstore/emitters/mock/inmem.go new file mode 100644 index 000000000..1d3043262 --- /dev/null +++ b/internal/logstore/emitters/mock/inmem.go @@ -0,0 +1,89 @@ +package mock + +import ( + "context" + "sync" + "time" + + "github.com/benbjohnson/clock" + + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +var _ logstore.UsageQuerier = (*InmemLogStorage)(nil) +var _ logstore.LogCleanupper = (*InmemLogStorage)(nil) + +type InmemLogStorage struct { + mux sync.Mutex + clock clock.Clock + emitted []*record + bulks []int +} + +func NewInMemoryStorage(clock clock.Clock) *InmemLogStorage { + return &InmemLogStorage{ + clock: clock, + emitted: make([]*record, 0), + bulks: make([]int, 0), + } +} + +func (l *InmemLogStorage) QuotaUnit() quota.Unit { + return quota.Unimplemented +} + +func (l *InmemLogStorage) Emit(_ context.Context, bulk []logstore.LogRecord) error { + if len(bulk) == 0 { + return nil + } + l.mux.Lock() + defer l.mux.Unlock() + for idx := range bulk { + l.emitted = append(l.emitted, bulk[idx].(*record)) + } + l.bulks = append(l.bulks, len(bulk)) + return nil +} + +func (l *InmemLogStorage) QueryUsage(_ context.Context, _ string, start time.Time) (uint64, error) { + l.mux.Lock() + defer l.mux.Unlock() + + var count uint64 + for _, r := range l.emitted { + if r.ts.After(start) { + count++ + } + } + return count, nil +} + +func (l *InmemLogStorage) Cleanup(_ context.Context, keep time.Duration) error { + l.mux.Lock() + defer l.mux.Unlock() + + clean := make([]*record, 0) + from := l.clock.Now().Add(-(keep + 1)) + for _, r := range l.emitted { + if r.ts.After(from) { + clean = append(clean, r) + } + } + l.emitted = clean + return nil +} + +func (l *InmemLogStorage) Bulks() []int { + l.mux.Lock() + defer l.mux.Unlock() + + return l.bulks +} + +func (l *InmemLogStorage) Len() int { + l.mux.Lock() + defer l.mux.Unlock() + + return len(l.emitted) +} diff --git a/internal/logstore/emitters/mock/record.go b/internal/logstore/emitters/mock/record.go new file mode 100644 index 000000000..be9fc6a1f --- /dev/null +++ b/internal/logstore/emitters/mock/record.go @@ -0,0 +1,25 @@ +package mock + +import ( + "time" + + "github.com/benbjohnson/clock" + + "github.com/zitadel/zitadel/internal/logstore" +) + +var _ logstore.LogRecord = (*record)(nil) + +func NewRecord(clock clock.Clock) *record { + return &record{ts: clock.Now()} +} + +type record struct { + ts time.Time + redacted bool +} + +func (r record) Normalize() logstore.LogRecord { + r.redacted = true + return &r +} diff --git a/internal/logstore/emitters/stdout/stdout.go b/internal/logstore/emitters/stdout/stdout.go new file mode 100644 index 000000000..7a2cd53e6 --- /dev/null +++ b/internal/logstore/emitters/stdout/stdout.go @@ -0,0 +1,23 @@ +package stdout + +import ( + "context" + "encoding/json" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/logstore" +) + +func NewStdoutEmitter() logstore.LogEmitter { + return logstore.LogEmitterFunc(func(ctx context.Context, bulk []logstore.LogRecord) error { + for idx := range bulk { + bytes, err := json.Marshal(bulk[idx]) + if err != nil { + return err + } + logging.WithFields("record", string(bytes)).Info("log record emitted") + } + return nil + }) +} diff --git a/internal/logstore/helpers_test.go b/internal/logstore/helpers_test.go new file mode 100644 index 000000000..90348db77 --- /dev/null +++ b/internal/logstore/helpers_test.go @@ -0,0 +1,83 @@ +package logstore_test + +import ( + "time" + + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +type emitterOption func(config *logstore.EmitterConfig) + +func emitterConfig(options ...emitterOption) *logstore.EmitterConfig { + cfg := &logstore.EmitterConfig{ + Enabled: true, + Keep: time.Hour, + CleanupInterval: time.Hour, + Debounce: &logstore.DebouncerConfig{ + MinFrequency: 0, + MaxBulkSize: 0, + }, + } + for _, opt := range options { + opt(cfg) + } + return cfg +} + +func withDebouncerConfig(config *logstore.DebouncerConfig) emitterOption { + return func(c *logstore.EmitterConfig) { + c.Debounce = config + } +} + +func withDisabled() emitterOption { + return func(c *logstore.EmitterConfig) { + c.Enabled = false + } +} + +func withCleanupping(keep, interval time.Duration) emitterOption { + return func(c *logstore.EmitterConfig) { + c.Keep = keep + c.CleanupInterval = interval + } +} + +type quotaOption func(config *quota.AddedEvent) + +func quotaConfig(quotaOptions ...quotaOption) quota.AddedEvent { + q := "a.AddedEvent{ + Amount: 90, + Limit: false, + ResetInterval: 90 * time.Second, + From: time.Unix(0, 0), + } + for _, opt := range quotaOptions { + opt(q) + } + return *q +} + +func withAmountAndInterval(n uint64) quotaOption { + return func(c *quota.AddedEvent) { + c.Amount = n + c.ResetInterval = time.Duration(n) * time.Second + } +} + +func withLimiting() quotaOption { + return func(c *quota.AddedEvent) { + c.Limit = true + } +} + +func repeat(value, times int) []int { + ints := make([]int, times) + for i := 0; i < times; i++ { + ints[i] = value + } + return ints +} + +func uint64Ptr(n uint64) *uint64 { return &n } diff --git a/internal/logstore/quotaqueriers/mock/noop.go b/internal/logstore/quotaqueriers/mock/noop.go new file mode 100644 index 000000000..fd3b0aa81 --- /dev/null +++ b/internal/logstore/quotaqueriers/mock/noop.go @@ -0,0 +1,28 @@ +package mock + +import ( + "context" + "time" + + "github.com/zitadel/zitadel/internal/logstore" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +var _ logstore.QuotaQuerier = (*inmemReporter)(nil) + +type inmemReporter struct { + config *quota.AddedEvent + startPeriod time.Time +} + +func NewNoopQuerier(quota *quota.AddedEvent, startPeriod time.Time) *inmemReporter { + return &inmemReporter{config: quota, startPeriod: startPeriod} +} + +func (i *inmemReporter) GetCurrentQuotaPeriod(context.Context, string, quota.Unit) (*quota.AddedEvent, time.Time, error) { + return i.config, i.startPeriod, nil +} + +func (*inmemReporter) GetDueQuotaNotifications(context.Context, *quota.AddedEvent, time.Time, uint64) ([]*quota.NotifiedEvent, error) { + return nil, nil +} diff --git a/internal/logstore/service.go b/internal/logstore/service.go new file mode 100644 index 000000000..a24a7fea0 --- /dev/null +++ b/internal/logstore/service.go @@ -0,0 +1,110 @@ +package logstore + +import ( + "context" + "math" + "time" + + "github.com/zitadel/logging" + + "github.com/zitadel/zitadel/internal/repository/quota" +) + +type QuotaQuerier interface { + GetCurrentQuotaPeriod(ctx context.Context, instanceID string, unit quota.Unit) (config *quota.AddedEvent, periodStart time.Time, err error) + GetDueQuotaNotifications(ctx context.Context, config *quota.AddedEvent, periodStart time.Time, used uint64) ([]*quota.NotifiedEvent, error) +} + +type UsageQuerier interface { + LogEmitter + QuotaUnit() quota.Unit + QueryUsage(ctx context.Context, instanceId string, start time.Time) (uint64, error) +} + +type UsageReporter interface { + Report(ctx context.Context, notifications []*quota.NotifiedEvent) (err error) +} + +type UsageReporterFunc func(context.Context, []*quota.NotifiedEvent) (err error) + +func (u UsageReporterFunc) Report(ctx context.Context, notifications []*quota.NotifiedEvent) (err error) { + return u(ctx, notifications) +} + +type Service struct { + usageQuerier UsageQuerier + quotaQuerier QuotaQuerier + usageReporter UsageReporter + enabledSinks []*emitter + sinkEnabled bool + reportingEnabled bool +} + +func New(quotaQuerier QuotaQuerier, usageReporter UsageReporter, usageQuerierSink *emitter, additionalSink ...*emitter) *Service { + var usageQuerier UsageQuerier + if usageQuerierSink != nil { + usageQuerier = usageQuerierSink.emitter.(UsageQuerier) + } + + svc := &Service{ + reportingEnabled: usageQuerierSink != nil && usageQuerierSink.enabled, + usageQuerier: usageQuerier, + quotaQuerier: quotaQuerier, + usageReporter: usageReporter, + } + + for _, s := range append([]*emitter{usageQuerierSink}, additionalSink...) { + if s != nil && s.enabled { + svc.enabledSinks = append(svc.enabledSinks, s) + } + } + + svc.sinkEnabled = len(svc.enabledSinks) > 0 + + return svc +} + +func (s *Service) Enabled() bool { + return s.sinkEnabled +} + +func (s *Service) Handle(ctx context.Context, record LogRecord) { + for _, sink := range s.enabledSinks { + logging.OnError(sink.Emit(ctx, record.Normalize())).WithField("record", record).Warn("failed to emit log record") + } +} + +func (s *Service) Limit(ctx context.Context, instanceID string) *uint64 { + var err error + defer func() { + logging.OnError(err).Warn("failed to check is usage should be limited") + }() + + if !s.reportingEnabled || instanceID == "" { + return nil + } + + quota, periodStart, err := s.quotaQuerier.GetCurrentQuotaPeriod(ctx, instanceID, s.usageQuerier.QuotaUnit()) + if err != nil || quota == nil { + return nil + } + + usage, err := s.usageQuerier.QueryUsage(ctx, instanceID, periodStart) + if err != nil { + return nil + } + + var remaining *uint64 + if quota.Limit { + r := uint64(math.Max(0, float64(quota.Amount)-float64(usage))) + remaining = &r + } + + notifications, err := s.quotaQuerier.GetDueQuotaNotifications(ctx, quota, periodStart, usage) + if err != nil { + return remaining + } + + err = s.usageReporter.Report(ctx, notifications) + return remaining +} diff --git a/internal/logstore/service_test.go b/internal/logstore/service_test.go new file mode 100644 index 000000000..ecf8d287a --- /dev/null +++ b/internal/logstore/service_test.go @@ -0,0 +1,312 @@ +// The library github.com/benbjohnson/clock fails when race is enabled +// https://github.com/benbjohnson/clock/issues/44 +//go:build !race + +package logstore_test + +import ( + "context" + "reflect" + "runtime" + "testing" + "time" + + "github.com/benbjohnson/clock" + + "github.com/zitadel/zitadel/internal/logstore" + emittermock "github.com/zitadel/zitadel/internal/logstore/emitters/mock" + quotaqueriermock "github.com/zitadel/zitadel/internal/logstore/quotaqueriers/mock" + "github.com/zitadel/zitadel/internal/repository/quota" +) + +const ( + tick = time.Second + ticks = 60 +) + +type args struct { + mainSink *logstore.EmitterConfig + secondarySink *logstore.EmitterConfig + config quota.AddedEvent +} + +type want struct { + enabled bool + remaining *uint64 + mainSink wantSink + secondarySink wantSink +} + +type wantSink struct { + bulks []int + len int +} + +func TestService(t *testing.T) { + // tests should run on a single thread + // important for deterministic results + beforeProcs := runtime.GOMAXPROCS(1) + defer runtime.GOMAXPROCS(beforeProcs) + + tests := []struct { + name string + args args + want want + }{{ + name: "max and min debouncing works", + args: args{ + mainSink: emitterConfig(withDebouncerConfig(&logstore.DebouncerConfig{ + MinFrequency: 1 * time.Minute, + MaxBulkSize: 60, + })), + secondarySink: emitterConfig(), + config: quotaConfig(), + }, + want: want{ + enabled: true, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(60, 1), + len: 60, + }, + secondarySink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + }, + }, { + name: "mixed debouncing works", + args: args{ + mainSink: emitterConfig(withDebouncerConfig(&logstore.DebouncerConfig{ + MinFrequency: 0, + MaxBulkSize: 6, + })), + secondarySink: emitterConfig(withDebouncerConfig(&logstore.DebouncerConfig{ + MinFrequency: 10 * time.Second, + MaxBulkSize: 0, + })), + config: quotaConfig(), + }, + want: want{ + enabled: true, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(6, 10), + len: 60, + }, + secondarySink: wantSink{ + bulks: repeat(10, 6), + len: 60, + }, + }, + }, { + name: "when disabling main sink, secondary sink still works", + args: args{ + mainSink: emitterConfig(withDisabled()), + secondarySink: emitterConfig(), + config: quotaConfig(), + }, + want: want{ + enabled: true, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(99, 0), + len: 0, + }, + secondarySink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + }, + }, { + name: "when all sink are disabled, the service is disabled", + args: args{ + mainSink: emitterConfig(withDisabled()), + secondarySink: emitterConfig(withDisabled()), + config: quotaConfig(), + }, + want: want{ + enabled: false, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(99, 0), + len: 0, + }, + secondarySink: wantSink{ + bulks: repeat(99, 0), + len: 0, + }, + }, + }, { + name: "cleanupping works", + args: args{ + mainSink: emitterConfig(withCleanupping(17*time.Second, 28*time.Second)), + secondarySink: emitterConfig(withDebouncerConfig(&logstore.DebouncerConfig{ + MinFrequency: 0, + MaxBulkSize: 15, + }), withCleanupping(5*time.Second, 47*time.Second)), + config: quotaConfig(), + }, + want: want{ + enabled: true, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(1, 60), + len: 21, + }, + secondarySink: wantSink{ + bulks: repeat(15, 4), + len: 18, + }, + }, + }, { + name: "when quota has a limit of 90, 30 are remaining", + args: args{ + mainSink: emitterConfig(), + secondarySink: emitterConfig(), + config: quotaConfig(withLimiting()), + }, + want: want{ + enabled: true, + remaining: uint64Ptr(30), + mainSink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + secondarySink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + }, + }, { + name: "when quota has a limit of 30, 0 are remaining", + args: args{ + mainSink: emitterConfig(), + secondarySink: emitterConfig(), + config: quotaConfig(withLimiting(), withAmountAndInterval(30)), + }, + want: want{ + enabled: true, + remaining: uint64Ptr(0), + mainSink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + secondarySink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + }, + }, { + name: "when quota has amount of 30 but is not limited, remaining is nil", + args: args{ + mainSink: emitterConfig(), + secondarySink: emitterConfig(), + config: quotaConfig(withAmountAndInterval(30)), + }, + want: want{ + enabled: true, + remaining: nil, + mainSink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + secondarySink: wantSink{ + bulks: repeat(1, 60), + len: 60, + }, + }, + }} + for _, tt := range tests { + runTest(t, tt.name, tt.args, tt.want) + } +} + +func runTest(t *testing.T, name string, args args, want want) bool { + return t.Run("Given over a minute, each second a log record is emitted", func(tt *testing.T) { + tt.Run(name, func(t *testing.T) { + ctx, clock, mainStorage, secondaryStorage, svc := given(t, args, want) + remaining := when(svc, ctx, clock) + then(t, mainStorage, secondaryStorage, remaining, want) + }) + }) +} + +func given(t *testing.T, args args, want want) (context.Context, *clock.Mock, *emittermock.InmemLogStorage, *emittermock.InmemLogStorage, *logstore.Service) { + ctx := context.Background() + clock := clock.NewMock() + + periodStart := time.Time{} + clock.Set(args.config.From) + + mainStorage := emittermock.NewInMemoryStorage(clock) + mainEmitter, err := logstore.NewEmitter(ctx, clock, args.mainSink, mainStorage) + if err != nil { + t.Errorf("expected no error but got %v", err) + } + secondaryStorage := emittermock.NewInMemoryStorage(clock) + secondaryEmitter, err := logstore.NewEmitter(ctx, clock, args.secondarySink, secondaryStorage) + if err != nil { + t.Errorf("expected no error but got %v", err) + } + + svc := logstore.New( + quotaqueriermock.NewNoopQuerier(&args.config, periodStart), + logstore.UsageReporterFunc(func(context.Context, []*quota.NotifiedEvent) error { return nil }), + mainEmitter, + secondaryEmitter) + + if svc.Enabled() != want.enabled { + t.Errorf("wantet service enabled to be %t but is %t", want.enabled, svc.Enabled()) + } + return ctx, clock, mainStorage, secondaryStorage, svc +} + +func when(svc *logstore.Service, ctx context.Context, clock *clock.Mock) *uint64 { + var remaining *uint64 + for i := 0; i < ticks; i++ { + svc.Handle(ctx, emittermock.NewRecord(clock)) + runtime.Gosched() + remaining = svc.Limit(ctx, "non-empty-instance-id") + clock.Add(tick) + } + time.Sleep(time.Millisecond) + runtime.Gosched() + return remaining +} + +func then(t *testing.T, mainStorage, secondaryStorage *emittermock.InmemLogStorage, remaining *uint64, want want) { + mainBulks := mainStorage.Bulks() + if !reflect.DeepEqual(want.mainSink.bulks, mainBulks) { + t.Errorf("wanted main storage to have bulks %v, but got %v", want.mainSink.bulks, mainBulks) + } + + mainLen := mainStorage.Len() + if !reflect.DeepEqual(want.mainSink.len, mainLen) { + t.Errorf("wanted main storage to have len %d, but got %d", want.mainSink.len, mainLen) + } + + secondaryBulks := secondaryStorage.Bulks() + if !reflect.DeepEqual(want.secondarySink.bulks, secondaryBulks) { + t.Errorf("wanted secondary storage to have bulks %v, but got %v", want.secondarySink.bulks, secondaryBulks) + } + + secondaryLen := secondaryStorage.Len() + if !reflect.DeepEqual(want.secondarySink.len, secondaryLen) { + t.Errorf("wanted secondary storage to have len %d, but got %d", want.secondarySink.len, secondaryLen) + } + + if remaining == nil && want.remaining == nil { + return + } + + if remaining == nil && want.remaining != nil || + remaining != nil && want.remaining == nil { + t.Errorf("wantet remaining nil %t but got %t", want.remaining == nil, remaining == nil) + return + } + if *remaining != *want.remaining { + t.Errorf("wantet remaining %d but got %d", *want.remaining, *remaining) + return + } +} diff --git a/internal/query/instance.go b/internal/query/instance.go index 09e4859e6..9e5ef8673 100644 --- a/internal/query/instance.go +++ b/internal/query/instance.go @@ -219,6 +219,10 @@ func (q *Queries) InstanceByHost(ctx context.Context, host string) (_ authz.Inst return scan(row) } +func (q *Queries) InstanceByID(ctx context.Context) (_ authz.Instance, err error) { + return q.Instance(ctx, true) +} + func (q *Queries) GetDefaultLanguage(ctx context.Context) language.Tag { instance, err := q.Instance(ctx, false) if err != nil { diff --git a/internal/query/projection/projection.go b/internal/query/projection/projection.go index 18462f620..20d263079 100644 --- a/internal/query/projection/projection.go +++ b/internal/query/projection/projection.go @@ -156,7 +156,6 @@ func Start() { func ApplyCustomConfig(customConfig CustomConfig) crdb.StatementHandlerConfig { return applyCustomConfig(projectionConfig, customConfig) - } func applyCustomConfig(config crdb.StatementHandlerConfig, customConfig CustomConfig) crdb.StatementHandlerConfig { diff --git a/internal/repository/quota/aggregate.go b/internal/repository/quota/aggregate.go new file mode 100644 index 000000000..0e17faed7 --- /dev/null +++ b/internal/repository/quota/aggregate.go @@ -0,0 +1,26 @@ +package quota + +import ( + "github.com/zitadel/zitadel/internal/eventstore" +) + +const ( + AggregateType = "quota" + AggregateVersion = "v1" +) + +type Aggregate struct { + eventstore.Aggregate +} + +func NewAggregate(id, instanceId, resourceOwner string) *Aggregate { + return &Aggregate{ + Aggregate: eventstore.Aggregate{ + Type: AggregateType, + Version: AggregateVersion, + ID: id, + InstanceID: instanceId, + ResourceOwner: resourceOwner, + }, + } +} diff --git a/internal/repository/quota/events.go b/internal/repository/quota/events.go new file mode 100644 index 000000000..e0f8a5fcd --- /dev/null +++ b/internal/repository/quota/events.go @@ -0,0 +1,205 @@ +package quota + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "github.com/zitadel/zitadel/internal/errors" + "github.com/zitadel/zitadel/internal/eventstore" + "github.com/zitadel/zitadel/internal/eventstore/repository" +) + +type Unit uint + +const ( + UniqueQuotaNameType = "quota_units" + UniqueQuotaNotificationIDType = "quota_notification" + eventTypePrefix = eventstore.EventType("quota.") + AddedEventType = eventTypePrefix + "added" + NotifiedEventType = eventTypePrefix + "notified" + RemovedEventType = eventTypePrefix + "removed" +) + +const ( + Unimplemented Unit = iota + RequestsAllAuthenticated + ActionsAllRunsSeconds +) + +func NewAddQuotaUnitUniqueConstraint(unit Unit) *eventstore.EventUniqueConstraint { + return eventstore.NewAddEventUniqueConstraint( + UniqueQuotaNameType, + strconv.FormatUint(uint64(unit), 10), + "Errors.Quota.AlreadyExists", + ) +} + +func NewRemoveQuotaNameUniqueConstraint(unit Unit) *eventstore.EventUniqueConstraint { + return eventstore.NewRemoveEventUniqueConstraint( + UniqueQuotaNameType, + strconv.FormatUint(uint64(unit), 10), + ) +} + +type AddedEvent struct { + eventstore.BaseEvent `json:"-"` + + Unit Unit `json:"unit"` + From time.Time `json:"from"` + ResetInterval time.Duration `json:"interval,omitempty"` + Amount uint64 `json:"amount"` + Limit bool `json:"limit"` + Notifications []*AddedEventNotification `json:"notifications,omitempty"` +} + +type AddedEventNotification struct { + ID string `json:"id"` + Percent uint16 `json:"percent"` + Repeat bool `json:"repeat,omitempty"` + CallURL string `json:"callUrl"` +} + +func (e *AddedEvent) Data() interface{} { + return e +} + +func (e *AddedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint { + return []*eventstore.EventUniqueConstraint{NewAddQuotaUnitUniqueConstraint(e.Unit)} +} + +func NewAddedEvent( + ctx context.Context, + aggregate *eventstore.Aggregate, + unit Unit, + from time.Time, + resetInterval time.Duration, + amount uint64, + limit bool, + notifications []*AddedEventNotification, +) *AddedEvent { + return &AddedEvent{ + BaseEvent: *eventstore.NewBaseEventForPush( + ctx, + aggregate, + AddedEventType, + ), + Unit: unit, + From: from, + ResetInterval: resetInterval, + Amount: amount, + Limit: limit, + Notifications: notifications, + } +} + +func AddedEventMapper(event *repository.Event) (eventstore.Event, error) { + e := &AddedEvent{ + BaseEvent: *eventstore.BaseEventFromRepo(event), + } + + err := json.Unmarshal(event.Data, e) + if err != nil { + return nil, errors.ThrowInternal(err, "ACTION-4n8vs", "unable to unmarshal quota added") + } + + return e, nil +} + +type NotifiedEvent struct { + eventstore.BaseEvent `json:"-"` + Unit Unit `json:"unit"` + ID string `json:"id"` + CallURL string `json:"callURL"` + PeriodStart time.Time `json:"periodStart"` + Threshold uint16 `json:"threshold"` + Usage uint64 `json:"usage"` +} + +func (e *NotifiedEvent) Data() interface{} { + return e +} + +func (e *NotifiedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint { + return nil +} + +func NewNotifiedEvent( + ctx context.Context, + aggregate *eventstore.Aggregate, + unit Unit, + id string, + callURL string, + periodStart time.Time, + threshold uint16, + usage uint64, +) *NotifiedEvent { + return &NotifiedEvent{ + BaseEvent: *eventstore.NewBaseEventForPush( + ctx, + aggregate, + NotifiedEventType, + ), + Unit: unit, + ID: id, + CallURL: callURL, + PeriodStart: periodStart, + Threshold: threshold, + Usage: usage, + } +} + +func NotifiedEventMapper(event *repository.Event) (eventstore.Event, error) { + e := &NotifiedEvent{ + BaseEvent: *eventstore.BaseEventFromRepo(event), + } + + err := json.Unmarshal(event.Data, e) + if err != nil { + return nil, errors.ThrowInternal(err, "ACTION-4n8vs", "unable to unmarshal quota notified") + } + + return e, nil +} + +type RemovedEvent struct { + eventstore.BaseEvent `json:"-"` + Unit Unit `json:"unit"` +} + +func (e *RemovedEvent) Data() interface{} { + return e +} + +func (e *RemovedEvent) UniqueConstraints() []*eventstore.EventUniqueConstraint { + return []*eventstore.EventUniqueConstraint{NewRemoveQuotaNameUniqueConstraint(e.Unit)} +} + +func NewRemovedEvent( + ctx context.Context, + aggregate *eventstore.Aggregate, + unit Unit, +) *RemovedEvent { + return &RemovedEvent{ + BaseEvent: *eventstore.NewBaseEventForPush( + ctx, + aggregate, + RemovedEventType, + ), + Unit: unit, + } +} + +func RemovedEventMapper(event *repository.Event) (eventstore.Event, error) { + e := &RemovedEvent{ + BaseEvent: *eventstore.BaseEventFromRepo(event), + } + + err := json.Unmarshal(event.Data, e) + if err != nil { + return nil, errors.ThrowInternal(err, "ACTION-4bReE", "unable to unmarshal quota removed") + } + + return e, nil +} diff --git a/internal/repository/quota/eventstore.go b/internal/repository/quota/eventstore.go new file mode 100644 index 000000000..1725b2bc4 --- /dev/null +++ b/internal/repository/quota/eventstore.go @@ -0,0 +1,11 @@ +package quota + +import ( + "github.com/zitadel/zitadel/internal/eventstore" +) + +func RegisterEventMappers(es *eventstore.Eventstore) { + es.RegisterFilterEventMapper(AggregateType, AddedEventType, AddedEventMapper). + RegisterFilterEventMapper(AggregateType, NotifiedEventType, NotifiedEventMapper). + RegisterFilterEventMapper(AggregateType, RemovedEventType, RemovedEventMapper) +} diff --git a/internal/static/i18n/de.yaml b/internal/static/i18n/de.yaml index 944c34180..29e00d82d 100644 --- a/internal/static/i18n/de.yaml +++ b/internal/static/i18n/de.yaml @@ -437,6 +437,27 @@ Errors: CloseRows: SQL Statement konnte nicht abgeschlossen werden SQLStatement: SQL Statement konnte nicht erstellt werden InvalidRequest: Anfrage ist ungültig + Quota: + AlreadyExists: Das Kontingent existiert bereits für diese Einheit + NotFound: Kontingent für diese Einheit nicht gefunden + Invalid: + CallURL: Aufruf URL für Kontingent ist ungültig + Percent: Kontingent Prozent is unter 1 + Unimplemented: Kontingente sind für diese Einheit nicht implementiert + Amount: Kontingent Menge ist kleiner als 1 + ResetInterval: Das Rücksetzungsintervall für das Kontingent ist kürzer als eine Minute + Noop: Ein unlimitiertes Kontingent ohne Benachrichtigungen hat keinen Effekt + Access: + Exhausted: Das Kontingent für authentifizierte Requests ist aufgebraucht + Execution: + Exhausted: Das Kontingent für Action Sekunden ist aufgebraucht + LogStore: + Access: + StorageFailed: Das Speichern des Access Logs in der Datenbank ist fehlgeschlagen + ScanFailed: Das Abfragen der verbrauchten authentifizierten Requests ist fehlgeschlagen + Execution: + StorageFailed: Das Speichern des Action Logs in der Datenbank ist fehlgeschlagen + ScanFailed: Das Abfragen der verbrauchten Actions Sekunden ist fehlgeschlagen AggregateTypes: action: Action @@ -446,6 +467,7 @@ AggregateTypes: project: Projekt user: Benutzer usergrant: Benutzerberechtigung + quota: Kontingent EventTypes: user: diff --git a/internal/static/i18n/en.yaml b/internal/static/i18n/en.yaml index 44eaaefb9..ab13bb453 100644 --- a/internal/static/i18n/en.yaml +++ b/internal/static/i18n/en.yaml @@ -437,6 +437,28 @@ Errors: CloseRows: SQL Statement could not be finished SQLStatement: SQL Statement could not be created InvalidRequest: Request is invalid + Quota: + AlreadyExists: Quota already exists for this unit + NotFound: Quota not found for this unit + Invalid: + CallURL: Quota call URL is invalid + Percent: Quota percent is lower than 1 + Unimplemented: Quotas are not implemented for this unit + Amount: Quota amount is lower than 1 + ResetInterval: Quota reset interval is shorter than a minute + Noop: An unlimited quota without notifications has no effect + Access: + Exhausted: The quota for authenticated requests is exhausted + Execution: + Exhausted: The quota for execution seconds is exhausted + LogStore: + Access: + StorageFailed: Storing access log to database failed + ScanFailed: Querying usage for authenticated requests failed + Execution: + StorageFailed: Storing action execution log to database failed + ScanFailed: Querying usage for action execution seconds failed + AggregateTypes: action: Action @@ -446,6 +468,7 @@ AggregateTypes: project: Project user: User usergrant: User grant + quota: Quota EventTypes: user: diff --git a/internal/static/i18n/fr.yaml b/internal/static/i18n/fr.yaml index 8cf12e540..e451dc36c 100644 --- a/internal/static/i18n/fr.yaml +++ b/internal/static/i18n/fr.yaml @@ -437,6 +437,27 @@ Errors: CloseRows: L'instruction SQL n'a pas pu être terminée SQLStatement: L'instruction SQL n'a pas pu être créée InvalidRequest: La requête n'est pas valide + Quota: + AlreadyExists: Contingent existe déjà pour cette unité + NotFound: Contingent non trouvé pour cette unité + Invalid: + CallURL: L'URL d'appel du contingent n'est pas valide + Percent: Pourcent du contingent est inférieure à 1 + Unimplemented: Les contingents ne sont pas implémentés pour cette unité + Amount: Quantité contingentée est inférieure à 1 + ResetInterval: L'intervalle de réinitialisation entre les contingents est inférieur à une minute + Noop: Un contingent illimité sans notifications n'a aucun effet + Access: + Exhausted: Le quota de requêtes authentifiées est épuisé + Execution: + Exhausted: Le quota de secondes d'action est épuisé + LogStore: + Access: + StorageFailed: L'enregistrement du journal d'accès dans la base de données a échoué + ScanFailed: L'interrogation des requêtes authentifiées consommées a échoué + Execution: + StorageFailed: L'enregistrement du journal d'action dans la base de données a échoué + ScanFailed: L'interrogation des secondes d'action consommées a échoué AggregateTypes: action: Action @@ -446,6 +467,7 @@ AggregateTypes: project: Projet user: Utilisateur usergrant: Subvention de l'utilisateur + quota: Contingent EventTypes: user: diff --git a/internal/static/i18n/it.yaml b/internal/static/i18n/it.yaml index e8645c8ad..91014813d 100644 --- a/internal/static/i18n/it.yaml +++ b/internal/static/i18n/it.yaml @@ -437,6 +437,27 @@ Errors: CloseRows: Lo statement SQL non può essere terminato SQLStatement: Lo statement SQL non può essere creato InvalidRequest: La richiesta non è valida + Quota: + AlreadyExists: La quota esiste già per questa unità + NotFound: Quota non trovata per questa unità + Invalid: + CallURL: L'URL di chiamata per la quota non è valido + Percent: La percentuale contingente è inferiore all'1 + Unimplemented: La quota non è implementata per questa unità + Amount: L'importo contingente è inferiore all'1 + ResetInterval: L'intervallo di reset contingente è inferiore a un minuto + Noop: Una quota illimitata senza notifiche non ha alcun effetto + Access: + Exhausted: La quota per le richieste autenticate è esaurita + Execution: + Exhausted: La quota per i secondi di azione è esaurita + LogStore: + Access: + StorageFailed: Il salvataggio del registro degli accessi nel database non è riuscito + ScanFailed: La query delle richieste autenticate utilizzate non è riuscita + Execution: + StorageFailed: Il salvataggio del registro delle azioni nel database non è riuscito + ScanFailed: La query dei secondi delle azioni utilizzate non è riuscita AggregateTypes: action: Azione @@ -446,6 +467,7 @@ AggregateTypes: project: Progetto user: Utente usergrant: Sovvenzione utente + quota: Quota EventTypes: user: diff --git a/internal/static/i18n/zh.yaml b/internal/static/i18n/zh.yaml index 84fd09a2a..a2ef6092b 100644 --- a/internal/static/i18n/zh.yaml +++ b/internal/static/i18n/zh.yaml @@ -437,6 +437,38 @@ Errors: CloseRows: SQL 语句无法完成 SQLStatement: 无法创建 SQL 语句 InvalidRequest: 请求无效 + Quota: + AlreadyExists: 这个单位的配额已经存在 + NotFound: 没有找到该单位的配额 + Invalid: + CallURL: 配额调用的URL是无效的 + Percent: 配额百分比低于1 + Unimplemented: 该单位没有实施配额 + Amount: 配额数量低于1 + ResetInterval: 配额重置时间间隔短于1分钟 + Noop: 没有通知的无限配额没有效果 + Access: + Exhausted: 认证请求的配额已用完 + Execution: + Exhausted: 行动秒数的配额已用完 + LogStore: + Access: + StorageFailed: 存储访问日志到数据库失败 + ScanFailed: 查询已认证请求的使用情况失败 + Execution: + StorageFailed: 将行动执行日志存储到数据库失败 + ScanFailed: Q查询动作执行秒数的使用情况失败 + +AggregateTypes: + action: 动作 + instance: 实例 + key_pair: 密钥对 + org: 组织 + project: 项目 + user: 用户 + usergrant: 用户授权 + quota: 配额 + EventTypes: user: added: 已添加用户 diff --git a/proto/zitadel/quota.proto b/proto/zitadel/quota.proto new file mode 100644 index 000000000..6323c78c5 --- /dev/null +++ b/proto/zitadel/quota.proto @@ -0,0 +1,43 @@ +syntax = "proto3"; + +import "protoc-gen-openapiv2/options/annotations.proto"; +import "validate/validate.proto"; + +package zitadel.quota.v1; + +option go_package ="github.com/zitadel/zitadel/pkg/grpc/quota"; + +enum Unit { + UNIT_UNIMPLEMENTED = 0; + /* The sum of all requests to the ZITADEL API with an authorization header, + excluding the following exceptions + - Calls to the System API + - Calls that cause internal server errors + - Failed authorizations + - Requests after the quota already exceeded + */ + UNIT_REQUESTS_ALL_AUTHENTICATED = 1; + // The sum of all actions run durations in seconds + UNIT_ACTIONS_ALL_RUN_SECONDS = 2; +} + +message Notification { + // The percentage relative to the quotas amount on which the call_url should be called. + uint32 percent = 1 [ + (validate.rules).uint32.gt = 0, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "The percentage relative to the quotas amount on which the call_url should be called."; + } + ]; + // If true, the call_url is called each time a factor of percentage is reached. + bool repeat = 2 [(grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "If true, the call_url is called each time a factor of percentage is reached."; + }]; + // The URL, which is called with HTTP method POST and a JSON payload with the properties "unit", "id" (notification id), "callURL", "periodStart", "threshold" and "usage". + string call_url = 3 [ + (validate.rules).string = {min_len: 1, max_len: 200}, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "The URL, which is called with HTTP method POST and a JSON payload with the properties \"unit\", \"id\" (notification id), \"callURL\", \"periodStart\", \"threshold\" and \"usage\"."; + } + ]; +} diff --git a/proto/zitadel/system.proto b/proto/zitadel/system.proto index 9b8c3edc6..d05e52646 100644 --- a/proto/zitadel/system.proto +++ b/proto/zitadel/system.proto @@ -4,10 +4,12 @@ import "zitadel/object.proto"; import "zitadel/options.proto"; import "zitadel/instance.proto"; import "zitadel/member.proto"; +import "zitadel/quota.proto"; import "zitadel/auth_n_key.proto"; import "google/api/annotations.proto"; import "google/protobuf/timestamp.proto"; +import "google/protobuf/duration.proto"; import "protoc-gen-openapiv2/options/annotations.proto"; import "validate/validate.proto"; @@ -162,7 +164,7 @@ service SystemService { }; } - // Removes a instances + // Removes an instance // This might take some time rpc RemoveInstance(RemoveInstanceRequest) returns (RemoveInstanceResponse) { option (google.api.http) = { @@ -353,6 +355,29 @@ service SystemService { }; }; } + + // Creates a new quota + rpc AddQuota(AddQuotaRequest) returns (AddQuotaResponse) { + option (google.api.http) = { + post: "/instances/{instance_id}/quotas" + body: "*" + }; + + option (zitadel.v1.auth_option) = { + permission: "authenticated"; + }; + } + + // Removes a quota + rpc RemoveQuota(RemoveQuotaRequest) returns (RemoveQuotaResponse) { + option (google.api.http) = { + delete: "/instances/{instance_id}/quotas/{unit}" + }; + + option (zitadel.v1.auth_option) = { + permission: "authenticated"; + }; + } } @@ -520,10 +545,62 @@ message GetUsageRequest { string instance_id = 1 [(validate.rules).string = {min_len: 1, max_len: 200}]; } -message GetUsageResponse { +message AddQuotaRequest { + string instance_id = 1 [(validate.rules).string = {min_len: 1, max_len: 200}]; + // the unit a quota should be imposed on + zitadel.quota.v1.Unit unit = 2 [ + (validate.rules).enum = {defined_only: true, not_in: [0]}, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "the unit a quota should be imposed on"; + } + ]; + // the starting time from which the current quota period is calculated from. This is relevant for querying the current usage. + google.protobuf.Timestamp from = 3 [ + (validate.rules).timestamp.required = true, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + example: "\"2019-04-01T08:45:00.000000Z\""; + description: "the starting time from which the current quota period is calculated from. This is relevant for querying the current usage."; + } + ]; + // the quota periods duration + google.protobuf.Duration reset_interval = 4 [ + (validate.rules).duration.required = true, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "the quota periods duration"; + } + ]; + // the quota amount of units + uint64 amount = 5 [ + (validate.rules).uint64.gt = 0, + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "the quota amount of units"; + } + ]; + // whether ZITADEL should block further usage when the configured amount is used + bool limit = 6 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "whether ZITADEL should block further usage when the configured amount is used"; + } + ]; + // the handlers, ZITADEL executes when certain quota percentages are reached + repeated zitadel.quota.v1.Notification notifications = 7 [ + (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = { + description: "the handlers, ZITADEL executes when certain quota percentages are reached"; + } + ]; +} + +message AddQuotaResponse { + zitadel.v1.ObjectDetails details = 1; +} + +message RemoveQuotaRequest { + string instance_id = 1 [(validate.rules).string = {min_len: 1, max_len: 200}]; + zitadel.quota.v1.Unit unit = 2; +} + +message RemoveQuotaResponse { zitadel.v1.ObjectDetails details = 1; - uint64 executed_requests = 2; - uint64 executed_action_mins = 3; } message ExistsDomainRequest {