From 3ca7147808b43e526576af7492cc840ce7042808 Mon Sep 17 00:00:00 2001 From: Miguel Cabrerizo <30386061+doncicuto@users.noreply.github.com> Date: Fri, 5 May 2023 10:25:02 +0200 Subject: [PATCH] fix: introduce measures to avoid bots crawling and indexing activities (#5728) * fix: 404 for robots.txt and meta robots tags * fix: add unit tests for robots txt and tag * fix: add meta tag robots none for login pages * fix: weird format issue in header.go * fix: add x-robots-tag=none to grpcwebserver * fix linting --------- Co-authored-by: Silvan Co-authored-by: Livio Spring --- cmd/start/start.go | 8 ++++++ console/src/index.html | 1 + internal/api/api.go | 2 ++ internal/api/grpc/server/gateway.go | 1 + internal/api/http/header.go | 1 + .../http/middleware/robots_tag_interceptor.go | 14 ++++++++++ .../middleware/robots_tag_interceptor_test.go | 24 ++++++++++++++++ internal/api/robots_txt/robots_txt.go | 19 +++++++++++++ internal/api/robots_txt/robots_txt_test.go | 28 +++++++++++++++++++ .../api/ui/login/static/templates/main.html | 1 + 10 files changed, 99 insertions(+) create mode 100644 internal/api/http/middleware/robots_tag_interceptor.go create mode 100644 internal/api/http/middleware/robots_tag_interceptor_test.go create mode 100644 internal/api/robots_txt/robots_txt.go create mode 100644 internal/api/robots_txt/robots_txt_test.go diff --git a/cmd/start/start.go b/cmd/start/start.go index 4f134b044..58571abc1 100644 --- a/cmd/start/start.go +++ b/cmd/start/start.go @@ -39,6 +39,7 @@ import ( http_util "github.com/zitadel/zitadel/internal/api/http" "github.com/zitadel/zitadel/internal/api/http/middleware" "github.com/zitadel/zitadel/internal/api/oidc" + "github.com/zitadel/zitadel/internal/api/robots_txt" "github.com/zitadel/zitadel/internal/api/saml" "github.com/zitadel/zitadel/internal/api/ui/console" "github.com/zitadel/zitadel/internal/api/ui/login" @@ -305,6 +306,13 @@ func startAPIs( return err } + // robots.txt handler + robotsTxtHandler, err := robots_txt.Start() + if err != nil { + return fmt.Errorf("unable to start robots txt handler: %w", err) + } + apis.RegisterHandlerOnPrefix(robots_txt.HandlerPrefix, robotsTxtHandler) + // TODO: Record openapi access logs? openAPIHandler, err := openapi.Start() if err != nil { diff --git a/console/src/index.html b/console/src/index.html index 14dc2035e..fe500d8d2 100644 --- a/console/src/index.html +++ b/console/src/index.html @@ -21,6 +21,7 @@ + diff --git a/internal/api/api.go b/internal/api/api.go index d634e5010..4efdb322a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -16,6 +16,7 @@ import ( internal_authz "github.com/zitadel/zitadel/internal/api/authz" "github.com/zitadel/zitadel/internal/api/grpc/server" http_util "github.com/zitadel/zitadel/internal/api/http" + http_mw "github.com/zitadel/zitadel/internal/api/http/middleware" "github.com/zitadel/zitadel/internal/api/ui/login" "github.com/zitadel/zitadel/internal/errors" "github.com/zitadel/zitadel/internal/logstore" @@ -167,6 +168,7 @@ func (a *API) routeGRPCWeb() { return true }), ) + a.router.Use(http_mw.RobotsTagHandler) a.router.NewRoute(). Methods(http.MethodPost, http.MethodOptions). MatcherFunc( diff --git a/internal/api/grpc/server/gateway.go b/internal/api/grpc/server/gateway.go index a4cbbbe61..faba43594 100644 --- a/internal/api/grpc/server/gateway.go +++ b/internal/api/grpc/server/gateway.go @@ -149,6 +149,7 @@ func addInterceptors(handler http.Handler, http1HostName string) http.Handler { handler = http_mw.CallDurationHandler(handler) handler = http1Host(handler, http1HostName) handler = http_mw.CORSInterceptor(handler) + handler = http_mw.RobotsTagHandler(handler) handler = http_mw.DefaultTelemetryHandler(handler) return http_mw.DefaultMetricsHandler(handler) } diff --git a/internal/api/http/header.go b/internal/api/http/header.go index 8402c271d..f2d3e19c1 100644 --- a/internal/api/http/header.go +++ b/internal/api/http/header.go @@ -23,6 +23,7 @@ const ( XUserAgent = "x-user-agent" XGrpcWeb = "x-grpc-web" XRequestedWith = "x-requested-with" + XRobotsTag = "x-robots-tag" IfNoneMatch = "If-None-Match" LastModified = "Last-Modified" Etag = "Etag" diff --git a/internal/api/http/middleware/robots_tag_interceptor.go b/internal/api/http/middleware/robots_tag_interceptor.go new file mode 100644 index 000000000..076a91cfa --- /dev/null +++ b/internal/api/http/middleware/robots_tag_interceptor.go @@ -0,0 +1,14 @@ +package middleware + +import ( + "net/http" + + http_utils "github.com/zitadel/zitadel/internal/api/http" +) + +func RobotsTagHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(http_utils.XRobotsTag, "none") + next.ServeHTTP(w, r) + }) +} diff --git a/internal/api/http/middleware/robots_tag_interceptor_test.go b/internal/api/http/middleware/robots_tag_interceptor_test.go new file mode 100644 index 000000000..7e30d4a9b --- /dev/null +++ b/internal/api/http/middleware/robots_tag_interceptor_test.go @@ -0,0 +1,24 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_RobotsTagInterceptor(t *testing.T) { + testHandler := func(w http.ResponseWriter, r *http.Request) {} + req := httptest.NewRequest(http.MethodGet, "/", nil) + recorder := httptest.NewRecorder() + + handler := RobotsTagHandler(http.HandlerFunc(testHandler)) + handler.ServeHTTP(recorder, req) + + res := recorder.Result() + exp := res.Header.Get("X-Robots-Tag") + assert.Equal(t, "none", exp) + + defer res.Body.Close() +} diff --git a/internal/api/robots_txt/robots_txt.go b/internal/api/robots_txt/robots_txt.go new file mode 100644 index 000000000..be487740d --- /dev/null +++ b/internal/api/robots_txt/robots_txt.go @@ -0,0 +1,19 @@ +package robots_txt + +import ( + "fmt" + "net/http" +) + +const ( + HandlerPrefix = "/robots.txt" +) + +func Start() (http.Handler, error) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/text") + fmt.Fprintf(w, "User-agent: *\nDisallow: /\n") + }) + return handler, nil +} diff --git a/internal/api/robots_txt/robots_txt_test.go b/internal/api/robots_txt/robots_txt_test.go new file mode 100644 index 000000000..ca70b6bc3 --- /dev/null +++ b/internal/api/robots_txt/robots_txt_test.go @@ -0,0 +1,28 @@ +package robots_txt + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_RobotsTxt(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/robots.txt", nil) + recorder := httptest.NewRecorder() + + handler, err := Start() + handler.ServeHTTP(recorder, req) + assert.Equal(t, nil, err) + + res := recorder.Result() + body, err := io.ReadAll(res.Body) + assert.Equal(t, nil, err) + + assert.Equal(t, 200, res.StatusCode) + assert.Equal(t, "User-agent: *\nDisallow: /\n", string(body)) + + defer res.Body.Close() +} diff --git a/internal/api/ui/login/static/templates/main.html b/internal/api/ui/login/static/templates/main.html index 08aac7c1b..04ec281eb 100644 --- a/internal/api/ui/login/static/templates/main.html +++ b/internal/api/ui/login/static/templates/main.html @@ -23,6 +23,7 @@ {{ .Title }} +