mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 17:25:48 +08:00
Andrey Antukh
cd4a4da0f2
* 🎉 Add telemetry anonymous event collection Rewrite the audit logging subsystem to support three operating modes and add anonymous telemetry event collection: Modes: - A (audit-log only): events persisted with full context - B (audit-log + telemetry): same as A, plus events are collected for telemetry shipping - C (telemetry-only): events stored anonymously with PII stripped, telemetry flag active, audit-log flag inactive Audit system refactoring (app.loggers.audit): - Replace qualified map keys (::audit/name etc.) with plain keywords - Rename submit! -> submit, insert! -> insert, prepare-event -> prepare-rpc-event - Add submit* as a lower-level public API - Add process-event dispatch function that handles all three modes and webhooks in a single tx-run! - Add :id to event schema (auto-generated if omitted) - Add filter-telemetry-props: anonymises event props per event type. Keeps UUID/boolean/number values; for login/identify events preserves lang, auth-backend, email-domain; for navigate events preserves route, file-id, team-id, page-id; instance-start trigger passes through. - Add filter-telemetry-context: retains only safe context keys. Backend: version, initiator, client-version, client-user-agent. Frontend: browser, os, locale, screen metrics, event-origin. - Timestamps truncated to day precision via ct/truncate for telemetry storage - PII stripped: props emptied, ip-addr zeroed, session-linking and access-token fields removed from context Config (app.config): - Derive :enable-telemetry flag from telemetry-enabled config option Email utilities (app.email): - Add email/clean and email/get-domain helper functions for domain extraction from email addresses Setup (app.setup): - Emit instance-start trigger event at system startup - Simplify handle-instance-id (remove read-only check) RPC layer (app.rpc): - wrap-audit now activates when :telemetry flag is set - Add :request-id to RPC params context for event correlation RPC commands (management, teams_invitations, verify_token, OIDC auth, webhooks): migrate all audit call sites to use the new plain-key API SREPL (app.srepl.main): - Migrate all audit/insert! calls to audit/insert with plain keys Telemetry task (app.tasks.telemetry): - Restructure legacy report into make-legacy-request; distinguish payload type as :telemetry-legacy-report - Add collect-and-send-audit-events: loop fetching up to 10,000 rows per iteration, encodes and sends each page, deletes on success, stops immediately on failure for retry - Add send-event-batch: POSTs fressian+zstd batch (base64 via blob/encode-str) to the telemetry endpoint with instance-id per event - Add gc-telemetry-events: enforces 100,000-row safety cap by dropping oldest rows first - Add delete-sent-events: deletes successfully shipped rows by id Blob utilities (app.util.blob): - Add encode-str/decode-str: combine fressian+zstd encoding with URL- safe base64 for JSON-safe string transport Database: - Add migration 0145: index on audit_log (source, created_at ASC) for efficient telemetry batch collection queries Frontend: - Always initialize event system regardless of :audit-log flag - Defer auth events (signin identify) to after profile is set - Refactor event subsystem for telemetry support Tests (21 test vars, 94 assertions in tasks-telemetry-test): - Cover all code paths: disabled/enabled telemetry, no-events no-op, happy-path batch send and delete, failure retention, payload anonymity, context stripping, timestamp day precision, batch encoding round-trip, multi-page iteration, GC cap enforcement, partial failure handling - blob encode-str/decode-str round-trip tests (14 test vars) - RPC audit integration tests (5 test vars) Signed-off-by: Andrey Antukh <niwi@niwi.nz> * 📎 Add pr feedback changes --------- Signed-off-by: Andrey Antukh <niwi@niwi.nz>
354 lines
12 KiB
Clojure
354 lines
12 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns app.main.data.auth
|
|
"Auth related data events"
|
|
(:require
|
|
[app.common.data :as d]
|
|
[app.common.exceptions :as ex]
|
|
[app.common.schema :as sm]
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.main.data.common :as dcm]
|
|
[app.main.data.event :as ev]
|
|
[app.main.data.notifications :as ntf]
|
|
[app.main.data.profile :as dp]
|
|
[app.main.data.team :as dtm]
|
|
[app.main.data.websocket :as ws]
|
|
[app.main.repo :as rp]
|
|
[app.main.router :as rt]
|
|
[app.util.i18n :as i18n :refer [tr]]
|
|
[app.util.storage :as storage]
|
|
[beicon.v2.core :as rx]
|
|
[potok.v2.core :as ptk]))
|
|
|
|
;; --- HELPERS
|
|
|
|
(defn is-authenticated?
|
|
[{:keys [id]}]
|
|
(and (uuid? id) (not= id uuid/zero)))
|
|
|
|
;; --- EVENT: login
|
|
|
|
(defn- logged-in
|
|
"This is the main event that is executed once we have logged in
|
|
profile. The profile can proceed from standard login or from
|
|
accepting invitation, or third party auth signup or singin."
|
|
[{:keys [props] :as profile}]
|
|
(letfn [(get-redirect-events [teams]
|
|
(if-let [token (:invitation-token profile)]
|
|
(rx/of (rt/nav :auth-verify-token {:token token}))
|
|
(if-let [redirect-href (:login-redirect storage/session)]
|
|
(binding [storage/*sync* true]
|
|
(swap! storage/session dissoc :login-redirect)
|
|
(if (= redirect-href (rt/get-current-href))
|
|
(rx/of (rt/reload true))
|
|
(rx/of (rt/nav-raw :href redirect-href))))
|
|
(if-let [file-id (get props :welcome-file-id)]
|
|
(rx/of (dcm/go-to-workspace
|
|
:file-id file-id
|
|
:team-id (:default-team-id profile))
|
|
(dp/update-profile-props {:welcome-file-id nil}))
|
|
|
|
(let [teams (into #{} (map :id) teams)
|
|
team-id (dtm/get-last-team-id)
|
|
team-id (if (and team-id (contains? teams team-id))
|
|
team-id
|
|
(:default-team-id profile))]
|
|
(rx/of (dcm/go-to-dashboard-recent {:team-id team-id})))))))]
|
|
|
|
(ptk/reify ::logged-in
|
|
ptk/WatchEvent
|
|
(watch [_ _ stream]
|
|
(cf/initialize-external-context-info)
|
|
|
|
|
|
(->> (rx/merge
|
|
(rx/of (dp/set-profile profile)
|
|
(ws/initialize)
|
|
(dtm/fetch-teams))
|
|
|
|
;; We schedule this event to be executed a bit later,
|
|
;; when the profile is already set
|
|
(->> (rx/of (ev/event {::ev/name "signin"
|
|
::ev/type "identify"
|
|
:id (:id profile)
|
|
:email (:email profile)
|
|
:auth-backend (:auth-backend profile)
|
|
:fullname (:fullname profile)
|
|
:is-muted (:is-muted profile)
|
|
:default-team-id (:default-team-id profile)
|
|
:default-project-id (:default-project-id profile)}))
|
|
(rx/observe-on :async))
|
|
|
|
|
|
(->> stream
|
|
(rx/filter (ptk/type? ::dtm/teams-fetched))
|
|
(rx/take 1)
|
|
(rx/map deref)
|
|
(rx/mapcat get-redirect-events)))
|
|
|
|
(rx/observe-on :async))))))
|
|
|
|
(declare login-from-register)
|
|
|
|
(defn login
|
|
[{:keys [email password invitation-token] :as data}]
|
|
(ptk/reify ::login
|
|
ptk/WatchEvent
|
|
(watch [_ _ stream]
|
|
(let [{:keys [on-error on-success]
|
|
:or {on-error rx/throw
|
|
on-success identity}} (meta data)
|
|
|
|
params {:email email
|
|
:password password
|
|
:invitation-token invitation-token}]
|
|
|
|
;; NOTE: We can't take the profile value from login because
|
|
;; there are cases when login is successful but the cookie is
|
|
;; not set properly (because of possible misconfiguration).
|
|
;; So, we proceed to make an additional call to fetch the
|
|
;; profile, and ensure that cookie is set correctly. If
|
|
;; profile fetch is successful, we mark the user logged in, if
|
|
;; the returned profile is an NOT authenticated profile, we
|
|
;; proceed to logout and show an error message.
|
|
|
|
(->> (rp/cmd! :login-with-password (d/without-nils params))
|
|
(rx/merge-map (fn [data]
|
|
(rx/merge
|
|
(rx/of (dp/fetch-profile))
|
|
(->> stream
|
|
(rx/filter dp/profile-fetched?)
|
|
(rx/take 1)
|
|
(rx/map deref)
|
|
(rx/filter (complement is-authenticated?))
|
|
(rx/tap on-error)
|
|
(rx/map #(ex/raise :type :authentication))
|
|
(rx/observe-on :async))
|
|
|
|
(->> stream
|
|
(rx/filter dp/profile-fetched?)
|
|
(rx/take 1)
|
|
(rx/map deref)
|
|
(rx/filter is-authenticated?)
|
|
(rx/map (fn [profile]
|
|
(with-meta (merge data profile)
|
|
{::ev/source "login"})))
|
|
(rx/tap on-success)
|
|
(rx/map logged-in)
|
|
(rx/observe-on :async)))))
|
|
(rx/catch on-error))))))
|
|
|
|
(def ^:private schema:login-with-ldap
|
|
[:map {:title "login-with-ldap"}
|
|
[:email ::sm/email]
|
|
[:password :string]])
|
|
|
|
(defn login-with-ldap
|
|
[params]
|
|
|
|
(assert (sm/check schema:login-with-ldap params))
|
|
|
|
(ptk/reify ::login-with-ldap
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(let [{:keys [on-error on-success]
|
|
:or {on-error rx/throw
|
|
on-success identity}} (meta params)]
|
|
(->> (rp/cmd! :login-with-ldap params)
|
|
(rx/tap on-success)
|
|
(rx/map (fn [profile]
|
|
(-> profile
|
|
(with-meta {::ev/source "login-with-ldap"})
|
|
(logged-in))))
|
|
(rx/catch on-error))))))
|
|
|
|
(def ^:private schema:login-with-sso
|
|
[:map {:title "login-with-sso"}
|
|
[:provider [:or :string ::sm/uuid]]])
|
|
|
|
(defn login-with-sso
|
|
"Start the SSO flow"
|
|
[params]
|
|
(assert (sm/check schema:login-with-sso params))
|
|
(ptk/reify ::login-with-sso
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(->> (rp/cmd! :login-with-oidc params)
|
|
(rx/map (fn [{:keys [redirect-uri] :as rsp}]
|
|
(if redirect-uri
|
|
(rt/nav-raw :uri redirect-uri)
|
|
(ex/raise :type :assertion
|
|
:code :unexpected-response
|
|
:hint "unexpected response from OIDC method"
|
|
:resp (pr-str rsp)))))
|
|
(rx/catch (fn [cause]
|
|
(let [{:keys [type code] :as error} (ex-data cause)]
|
|
(if (and (= type :restriction)
|
|
(= code :provider-not-configured))
|
|
(rx/of (ntf/error (tr "errors.auth-provider-not-configured")))
|
|
(rx/throw cause)))))))))
|
|
|
|
(defn login-from-token
|
|
"Used mainly as flow continuation after token validation."
|
|
[{:keys [profile] :as tdata}]
|
|
(ptk/reify ::login-from-token
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(->> (dp/on-fetch-profile-success profile)
|
|
(rx/map (fn [profile]
|
|
(logged-in (with-meta profile {::ev/source "login-with-token"}))))
|
|
;; NOTE: we need this to be asynchronous because the effect
|
|
;; should be called before proceed with the login process
|
|
(rx/observe-on :async)))))
|
|
|
|
(defn login-from-register
|
|
"Event used mainly for mark current session as logged-in in after the
|
|
user successfully registered using third party auth provider (in this
|
|
case we dont need to verify the email)."
|
|
[]
|
|
(ptk/reify ::login-from-register
|
|
ptk/WatchEvent
|
|
(watch [_ _ stream]
|
|
(rx/merge
|
|
(rx/of (dp/fetch-profile))
|
|
(->> stream
|
|
(rx/filter dp/profile-fetched?)
|
|
(rx/take 1)
|
|
(rx/map deref)
|
|
(rx/filter is-authenticated?)
|
|
(rx/map (fn [profile]
|
|
(with-meta profile
|
|
{::ev/source "register"})))
|
|
(rx/map logged-in)
|
|
(rx/observe-on :async))))))
|
|
|
|
;; --- EVENT: logout
|
|
|
|
(defn logged-out
|
|
[{:keys [redirect-uri]}]
|
|
(ptk/reify ::logged-out
|
|
ptk/UpdateEvent
|
|
(update [_ state]
|
|
(select-keys state [:route :router :session-id :history]))
|
|
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(if redirect-uri
|
|
(->> (rx/of (rt/nav-raw :uri (str redirect-uri)))
|
|
(rx/observe-on :async))
|
|
|
|
(rx/merge
|
|
;; NOTE: We need the `effect` of the current event to be
|
|
;; executed before the redirect.
|
|
(->> (rx/of (rt/nav :auth-login))
|
|
(rx/observe-on :async))
|
|
(rx/of (ws/finalize)))))
|
|
|
|
ptk/EffectEvent
|
|
(effect [_ _ _]
|
|
;; We prefer to keek some stuff in the storage like the current-team-id and the profile
|
|
(swap! storage/user (constantly {})))))
|
|
|
|
(defn logout
|
|
[]
|
|
(ptk/reify ::logout
|
|
ev/Event
|
|
(-data [_] {})
|
|
|
|
ptk/WatchEvent
|
|
(watch [_ state _]
|
|
(let [profile-id (:profile-id state)]
|
|
(->> (rx/interval 500)
|
|
(rx/take 1)
|
|
(rx/mapcat (fn [_]
|
|
(->> (rp/cmd! :logout {:profile-id profile-id})
|
|
(rx/delay-at-least 300)
|
|
(rx/catch (constantly (rx/of nil))))))
|
|
(rx/map logged-out))))))
|
|
|
|
;; --- Update Profile
|
|
|
|
(def ^:private
|
|
schema:request-profile-recovery
|
|
[:map {:title "request-profile-recovery" :closed true}
|
|
[:email ::sm/email]])
|
|
|
|
(defn request-profile-recovery
|
|
[data]
|
|
|
|
(assert (sm/check schema:request-profile-recovery data))
|
|
|
|
(ptk/reify ::request-profile-recovery
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(let [{:keys [on-error on-success]
|
|
:or {on-error rx/throw
|
|
on-success identity}} (meta data)]
|
|
|
|
(->> (rp/cmd! :request-profile-recovery data)
|
|
(rx/tap on-success)
|
|
(rx/catch on-error))))))
|
|
|
|
;; --- EVENT: recover-profile (Password)
|
|
|
|
(def ^:private
|
|
schema:recover-profile
|
|
[:map {:title "recover-profile" :closed true}
|
|
[:password :string]
|
|
[:token :string]])
|
|
|
|
(defn recover-profile
|
|
[data]
|
|
(assert (sm/check schema:recover-profile data))
|
|
|
|
(ptk/reify ::recover-profile
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(let [{:keys [on-error on-success]
|
|
:or {on-error rx/throw
|
|
on-success identity}} (meta data)]
|
|
(->> (rp/cmd! :recover-profile data)
|
|
(rx/tap on-success)
|
|
(rx/catch on-error))))))
|
|
|
|
;; --- EVENT: crete-demo-profile
|
|
|
|
(defn create-demo-profile
|
|
[]
|
|
(ptk/reify ::create-demo-profile
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(->> (rp/cmd! :create-demo-profile {})
|
|
(rx/map login)))))
|
|
|
|
(defn show-redirect-error
|
|
"A helper event that interprets the OIDC redirect errors on the URI
|
|
and shows an appropriate error message using the notification
|
|
banners."
|
|
[error]
|
|
(ptk/reify ::show-redirect-error
|
|
ptk/WatchEvent
|
|
(watch [_ _ _]
|
|
(when-let [hint (case error
|
|
"registration-disabled"
|
|
(tr "errors.registration-disabled")
|
|
"profile-blocked"
|
|
(tr "errors.profile-blocked")
|
|
"auth-provider-not-allowed"
|
|
(tr "errors.auth-provider-not-allowed")
|
|
"email-domain-not-allowed"
|
|
(tr "errors.email-domain-not-allowed")
|
|
|
|
;; We explicitly do not show any error here, it a explicit user operation.
|
|
"unable-to-auth"
|
|
nil
|
|
|
|
(tr "errors.generic"))]
|
|
|
|
(rx/of (ntf/warn hint))))))
|