{
  "page": {
    "id": "gvm7j1jbqtxh",
    "name": "Tonkean",
    "url": "https://status.tonkean.com"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"Communication Integrations","created_at":"2022-09-08T05:52:37.128-07:00","updated_at":"2026-05-24T05:00:07.363-07:00","position":6,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"vfpbpkykmwm9","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"Identity Provider Integrations","created_at":"2022-09-08T06:05:21.615-07:00","updated_at":"2026-05-24T05:00:07.390-07:00","position":7,"description":"Issues that impact the ability of administrators to configure and integrate authentication services with an external identity provider (such as Okta and Active Directory)","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"s899cxwtkm8f","group_id":null,"page_id":"gvm7j1jbqtxh"},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","group_id":null,"page_id":"gvm7j1jbqtxh"}],
  "incidents": [{"name":"Scheduled maintenance","status":"scheduled","created_at":"2026-09-02T06:54:25.826-07:00","updated_at":"2026-09-02T06:54:25.980-07:00","monitoring_at":null,"resolved_at":null,"impact":"maintenance","shortlink":"https://stspg.io/4jzvm35q01tx","scheduled_for":"2026-10-04T01:00:00.000-07:00","scheduled_until":"2026-10-04T09:00:00.000-07:00","scheduled_remind_prior":true,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-09-02T06:54:25.807-07:00","reminder_intervals":null,"id":"vnm59gr6lm4z","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"scheduled","body":"We will be undergoing scheduled maintenance during this time.","created_at":"2026-09-02T06:54:25.976-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-02T06:54:25.976-07:00","display_at":"2026-09-02T06:54:25.976-07:00","deliver_notifications":true,"tweet_id":null,"id":"6d2qxfp0qfz3","incident_id":"vnm59gr6lm4z","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Communication Integrations","created_at":"2022-09-08T05:52:37.128-07:00","updated_at":"2026-05-24T05:00:07.363-07:00","position":6,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"vfpbpkykmwm9","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Identity Provider Integrations","created_at":"2022-09-08T06:05:21.615-07:00","updated_at":"2026-05-24T05:00:07.390-07:00","position":7,"description":"Issues that impact the ability of administrators to configure and integrate authentication services with an external identity provider (such as Okta and Active Directory)","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"s899cxwtkm8f","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"High errors rate in workflow execution + slowness in app loading","status":"resolved","created_at":"2026-06-25T11:57:53.820-07:00","updated_at":"2026-07-01T02:24:35.549-07:00","monitoring_at":null,"resolved_at":"2026-06-25T13:15:02.873-07:00","impact":"minor","shortlink":"https://stspg.io/4417l9dyk8jp","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-06-25T11:57:53.809-07:00","reminder_intervals":null,"id":"2f39k6l47qff","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"All workflows and interfaces are back to normal.","created_at":"2026-06-25T13:15:02.873-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-25T13:15:02.873-07:00","display_at":"2026-06-25T13:15:02.873-07:00","deliver_notifications":true,"tweet_id":null,"id":"f8zvtswqnzsp","incident_id":"2f39k6l47qff","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"degraded_performance","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"degraded_performance","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We are seeing higher error rates and slow workflow execution times in production. We are investigating the root cause.","created_at":"2026-06-25T11:57:53.903-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-25T11:57:53.903-07:00","display_at":"2026-06-25T11:57:53.903-07:00","deliver_notifications":true,"tweet_id":null,"id":"cqr9p8svxy49","incident_id":"2f39k6l47qff","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"degraded_performance"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"degraded_performance"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":"On June 24–25, 2026, between approximately 22:00 UTC on June 24 and 19:37 UTC on June 25, users experienced slowness and failures when opening certain views that rely on connected data sources. Fields linked to those data sources did not populate, and in some cases, views failed to load. We recognize the disruption this caused and appreciate your patience.\n\nThe issue was first reported at 23:10 UTC on June 24. By 23:14 UTC, our engineering team had identified the cause: a separate project was generating an unusually large number of slow-running background queries that overwhelmed our shared search service, causing failures across connected data source fields. Service was confirmed restored by 19:37 UTC on June 25. No data was lost.\n\nThe primary driver was a configuration error in a separate project. A connected data source had grown significantly, and due to the misconfiguration, every record in that source was treated as newly changed on every sync — causing the same data to be re-processed continuously and flooding the shared service. An existing safeguard that would have limited the impact had also been removed the prior day due to an unrelated bug.\n\nWe corrected the misconfiguration to stop the excessive re-processing, and we are restoring the removed safeguard. Going forward, we are adding monitoring to detect and alert on this type of overload early, improving how we handle large data-source syncs so they do not flood shared services, and reviewing configuration defaults to prevent similar errors. We remain committed to the reliability of the Tonkean platform and to ensuring this type of incident does not recur.","postmortem_body_last_updated_at":"2026-07-01T02:24:35.534-07:00","postmortem_ignored":false,"postmortem_published_at":"2026-07-01T02:24:35.534-07:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"AI Agents are not responding","status":"resolved","created_at":"2026-06-19T02:07:40.805-07:00","updated_at":"2026-06-24T05:08:00.599-07:00","monitoring_at":null,"resolved_at":"2026-06-19T02:09:32.510-07:00","impact":"major","shortlink":"https://stspg.io/cbzxy5qc7dml","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-06-19T02:07:40.797-07:00","reminder_intervals":null,"id":"k37ndgq9wsmd","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"Incident is resolve, AI Agents are back working and responding.","created_at":"2026-06-19T02:09:32.510-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-19T02:09:32.510-07:00","display_at":"2026-06-19T02:09:32.510-07:00","deliver_notifications":true,"tweet_id":null,"id":"n5sh2m9dymzn","incident_id":"k37ndgq9wsmd","custom_tweet":null,"affected_components":[{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"partial_outage","new_status":"operational"}]},{"status":"identified","body":"We're seeing an issue with agents not responding at all at the moment. We've identified the root cause, and are working on a fix.","created_at":"2026-06-19T02:07:40.882-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-19T02:07:40.882-07:00","display_at":"2026-06-19T02:07:40.882-07:00","deliver_notifications":true,"tweet_id":null,"id":"hh24n6sk74tw","incident_id":"k37ndgq9wsmd","custom_tweet":null,"affected_components":[{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":"On 19 June 2026, between 02:47 and 09:08 UTC, all AI agents were unavailable across Tonkean’s production environments. Agents did not respond to requests during this window, interrupting active work and demonstrations. We recognize the significance of this disruption and appreciate your patience.   The outage was caused by Tonkean’s AI provider account reaching its usage and spending limits, which resulted in every agent request being rejected. Two factors extended the impact: the account’s quota had been consumed more quickly than anticipated, and an outstanding account balance had to be cleared before the quota could be restored — even after we had raised the limit. An automated alert had fired the previous afternoon, warning of the approaching limit, but it posted silently and was not acted on. Service was restored at 09:08 UTC after the balance was cleared and the limit raised. No data was lost.   To prevent recurrence, we are wiring the existing alert to actively page the on-call team and billing account owners rather than posting silently, and adding a proactive alert at roughly 90% of the spend limit.\n\nWe remain committed to the reliability of the Tonkean platform and to ensuring the gaps identified here are fully closed.","postmortem_body_last_updated_at":"2026-06-24T05:08:00.594-07:00","postmortem_ignored":false,"postmortem_published_at":"2026-06-23T07:41:27.341-07:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Scheduled maintenance","status":"completed","created_at":"2026-05-10T04:01:12.852-07:00","updated_at":"2026-05-24T05:00:07.563-07:00","monitoring_at":null,"resolved_at":"2026-05-24T05:00:07.514-07:00","impact":"maintenance","shortlink":"https://stspg.io/01hhmpmmnlqz","scheduled_for":"2026-05-24T02:00:00.000-07:00","scheduled_until":"2026-05-24T05:00:00.000-07:00","scheduled_remind_prior":true,"scheduled_reminded_at":"2026-05-24T01:00:10.290-07:00","impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-05-10T04:01:12.841-07:00","reminder_intervals":null,"id":"bzzmzlzx4dcd","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2026-05-24T05:00:07.514-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-24T05:00:07.514-07:00","display_at":"2026-05-24T05:00:07.514-07:00","deliver_notifications":true,"tweet_id":null,"id":"s8p20789hn2f","incident_id":"bzzmzlzx4dcd","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"under_maintenance","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"under_maintenance","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"under_maintenance","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-05-24T02:00:07.420-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-24T02:00:07.420-07:00","display_at":"2026-05-24T02:00:07.420-07:00","deliver_notifications":true,"tweet_id":null,"id":"p563z7ym0212","incident_id":"bzzmzlzx4dcd","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"under_maintenance"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"under_maintenance"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"under_maintenance"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2026-05-24T01:00:10.039-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-24T01:00:10.039-07:00","display_at":"2026-05-24T01:00:10.039-07:00","deliver_notifications":true,"tweet_id":null,"id":"kf91dfc5zt1j","incident_id":"bzzmzlzx4dcd","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"We will be undergoing scheduled maintenance during this time.","created_at":"2026-05-10T04:01:12.919-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-10T04:01:12.919-07:00","display_at":"2026-05-10T04:01:12.919-07:00","deliver_notifications":true,"tweet_id":null,"id":"px4zqrt06s19","incident_id":"bzzmzlzx4dcd","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Communication Integrations","created_at":"2022-09-08T05:52:37.128-07:00","updated_at":"2026-05-24T05:00:07.363-07:00","position":6,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"vfpbpkykmwm9","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Identity Provider Integrations","created_at":"2022-09-08T06:05:21.615-07:00","updated_at":"2026-05-24T05:00:07.390-07:00","position":7,"description":"Issues that impact the ability of administrators to configure and integrate authentication services with an external identity provider (such as Okta and Active Directory)","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"s899cxwtkm8f","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Sharepoint data sources errors","status":"resolved","created_at":"2026-05-15T08:14:27.809-07:00","updated_at":"2026-05-17T07:42:47.300-07:00","monitoring_at":"2026-05-15T08:30:37.428-07:00","resolved_at":"2026-05-15T12:26:29.057-07:00","impact":"critical","shortlink":"https://stspg.io/l0n320gcm11c","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-05-15T08:14:27.800-07:00","reminder_intervals":null,"id":"hmjcshzckjc8","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2026-05-15T12:26:29.057-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-15T12:26:29.057-07:00","display_at":"2026-05-15T12:26:29.057-07:00","deliver_notifications":true,"tweet_id":null,"id":"5mtg2708wrsn","incident_id":"hmjcshzckjc8","custom_tweet":null,"affected_components":[{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"We are continuing to monitor for any further issues.","created_at":"2026-05-15T08:41:31.876-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-15T08:41:31.876-07:00","display_at":"2026-05-15T08:41:31.876-07:00","deliver_notifications":true,"tweet_id":null,"id":"q11drgf4xhck","incident_id":"hmjcshzckjc8","custom_tweet":null,"affected_components":[{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"major_outage","new_status":"operational"}]},{"status":"monitoring","body":"A fix was implemented and we are currently monitoring the situation.","created_at":"2026-05-15T08:30:37.428-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-15T08:30:37.428-07:00","display_at":"2026-05-15T08:30:37.428-07:00","deliver_notifications":true,"tweet_id":null,"id":"k921124w0c8t","incident_id":"hmjcshzckjc8","custom_tweet":null,"affected_components":[{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"major_outage","new_status":"major_outage"}]},{"status":"identified","body":"We discovered an error with Sharepoint data sources breaking due to an invalid key error. We found the root cause and are working on a fix.","created_at":"2026-05-15T08:14:27.905-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-05-15T08:14:27.905-07:00","display_at":"2026-05-15T08:14:27.905-07:00","deliver_notifications":true,"tweet_id":null,"id":"f65zxxwvs7z5","incident_id":"hmjcshzckjc8","custom_tweet":null,"affected_components":[{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"major_outage"}]}],"postmortem_body":"We experienced a disruption to its SharePoint integration between May 14–15, 2026, caused by the expiration of authentication credentials used to connect to Microsoft SharePoint services. The issue affected customers using our shared SharePoint integration, causing failed SharePoint actions during the outage window. The first authentication failures were logged on May 14 at approximately 19:37 UTC. The incident was formally reported on May 15 at 17:28 UTC following the identification of multiple customer issues.  \nInvestigation began at 17:45 UTC, and expired Azure AD client secrets were identified as the root cause. Updated credentials were generated and deployed beginning at 18:01 UTC. Additional remediation steps and environment restarts were completed between 18:05 and 18:10 UTC to ensure the updated credentials were fully applied. Customers confirmed successful reconnection to SharePoint integrations following the fix.  \nTo reduce the likelihood of recurrence, we are enhancing our monitoring and alerting for credential expiration, expanding audits of integration authentication across environments, refining credential rotation procedures, and continuing efforts to further automate integration credential management workflows.","postmortem_body_last_updated_at":"2026-05-17T07:42:47.281-07:00","postmortem_ignored":false,"postmortem_published_at":"2026-05-17T07:42:47.281-07:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Scheduled maintenance","status":"completed","created_at":"2026-04-19T01:02:53.268-07:00","updated_at":"2026-04-26T02:22:54.161-07:00","monitoring_at":null,"resolved_at":"2026-04-26T02:22:54.142-07:00","impact":"maintenance","shortlink":"https://stspg.io/5fqgs5nbnc14","scheduled_for":"2026-04-26T01:00:00.000-07:00","scheduled_until":"2026-04-26T06:00:00.000-07:00","scheduled_remind_prior":true,"scheduled_reminded_at":"2026-04-26T00:00:11.958-07:00","impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-04-19T01:02:53.255-07:00","reminder_intervals":null,"id":"r2vj8gdzhh4c","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2026-04-26T02:22:54.142-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-26T02:22:54.142-07:00","display_at":"2026-04-26T02:22:54.142-07:00","deliver_notifications":true,"tweet_id":null,"id":"n6hwgqfkps48","incident_id":"r2vj8gdzhh4c","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"under_maintenance","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"under_maintenance","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"under_maintenance","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-04-26T01:00:07.260-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-26T01:00:07.260-07:00","display_at":"2026-04-26T01:00:07.260-07:00","deliver_notifications":true,"tweet_id":null,"id":"btls4fxpm9cp","incident_id":"r2vj8gdzhh4c","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"under_maintenance"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"under_maintenance"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"under_maintenance"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2026-04-26T00:00:11.764-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-26T00:00:11.764-07:00","display_at":"2026-04-26T00:00:11.764-07:00","deliver_notifications":true,"tweet_id":null,"id":"qrzxg4wzg075","incident_id":"r2vj8gdzhh4c","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"We will be undergoing scheduled maintenance during this time.","created_at":"2026-04-19T01:02:53.350-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-19T01:02:53.350-07:00","display_at":"2026-04-19T01:02:53.350-07:00","deliver_notifications":true,"tweet_id":null,"id":"1w19q9kz18f3","incident_id":"r2vj8gdzhh4c","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Communication Integrations","created_at":"2022-09-08T05:52:37.128-07:00","updated_at":"2026-05-24T05:00:07.363-07:00","position":6,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"vfpbpkykmwm9","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Identity Provider Integrations","created_at":"2022-09-08T06:05:21.615-07:00","updated_at":"2026-05-24T05:00:07.390-07:00","position":7,"description":"Issues that impact the ability of administrators to configure and integrate authentication services with an external identity provider (such as Okta and Active Directory)","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"s899cxwtkm8f","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Slowness in processing of Tonkean items, actions and interfaces","status":"resolved","created_at":"2026-04-16T01:33:44.791-07:00","updated_at":"2026-04-16T06:52:02.480-07:00","monitoring_at":null,"resolved_at":"2026-04-16T02:23:49.224-07:00","impact":"minor","shortlink":"https://stspg.io/p60j3hpwg63w","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-04-16T01:33:44.785-07:00","reminder_intervals":null,"id":"h172kb3c6gfv","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2026-04-16T02:23:49.224-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-16T02:23:49.224-07:00","display_at":"2026-04-16T02:23:49.224-07:00","deliver_notifications":true,"tweet_id":null,"id":"r2n3b87fz5qs","incident_id":"h172kb3c6gfv","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"degraded_performance","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"degraded_performance","new_status":"operational"}]},{"status":"investigating","body":"We're investigating a slowness in processing of Tonkean items, actions and interfaces","created_at":"2026-04-16T01:33:44.884-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-16T01:33:44.884-07:00","display_at":"2026-04-16T01:33:44.884-07:00","deliver_notifications":true,"tweet_id":null,"id":"vng2gqszs8sg","incident_id":"h172kb3c6gfv","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"degraded_performance"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":"On April 15, 2026 at 19:00 \\(UTC\\+3\\), a configuration issue triggered a loop within an automation module, causing repeated item creation and excessive system activity. This led to a significant increase in load and degraded platform performance.\n\nCustomers experienced elevated latency, including slower interface loading, delays when progressing through processes, and slower execution of automated actions. No data loss occurred.\n\nThe issue was identified within approximately one hour. Mitigation actions included stopping the loop and clearing overloaded processing queues. Full service was restored for all customers by April 16 at 12:23 \\(UTC\\+3\\).\n\nThe root cause was a trigger configuration that continuously re-executed under certain conditions, resulting in unbounded item creation and system load.\n\nTo prevent recurrence, we are implementing safeguards including rate limits on actions and item creation, defined execution quotas at the module and solution level, and automatic halting of processes when limits are exceeded with visibility in execution history.","postmortem_body_last_updated_at":"2026-04-16T06:52:02.470-07:00","postmortem_ignored":false,"postmortem_published_at":"2026-04-16T06:52:02.470-07:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Slow processing of Tonkean items and access to the Tonkean app","status":"resolved","created_at":"2026-04-15T10:32:02.907-07:00","updated_at":"2026-04-16T06:52:30.725-07:00","monitoring_at":"2026-04-15T14:11:11.621-07:00","resolved_at":"2026-04-15T14:22:07.715-07:00","impact":"major","shortlink":"https://stspg.io/jrcfzwpml7jw","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-04-15T10:32:02.903-07:00","reminder_intervals":null,"id":"p2lkkvcgjv86","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2026-04-15T14:22:07.715-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T14:22:07.715-07:00","display_at":"2026-04-15T14:22:07.715-07:00","deliver_notifications":true,"tweet_id":null,"id":"cx37wzl1mz4p","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"degraded_performance","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"operational"}]},{"status":"monitoring","body":"We are continuing to monitor for any further issues.","created_at":"2026-04-15T14:16:44.056-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T14:16:44.056-07:00","display_at":"2026-04-15T14:16:44.056-07:00","deliver_notifications":true,"tweet_id":null,"id":"xx1txn2n9z46","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"partial_outage","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"partial_outage","new_status":"degraded_performance"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"partial_outage","new_status":"operational"}]},{"status":"monitoring","body":"A fix has been implemented and we are monitoring the results.","created_at":"2026-04-15T14:11:11.621-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T14:11:11.621-07:00","display_at":"2026-04-15T14:11:11.621-07:00","deliver_notifications":true,"tweet_id":null,"id":"7jxgpc4zgyyf","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"partial_outage","new_status":"partial_outage"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"identified","body":"The issue has been identified and a fix is being implemented.","created_at":"2026-04-15T13:32:19.596-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T13:32:19.596-07:00","display_at":"2026-04-15T13:32:19.596-07:00","deliver_notifications":true,"tweet_id":null,"id":"btzrc18krg9n","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"partial_outage","new_status":"partial_outage"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are continuing to investigate this issue.","created_at":"2026-04-15T12:33:44.124-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T12:33:44.124-07:00","display_at":"2026-04-15T12:33:44.124-07:00","deliver_notifications":true,"tweet_id":null,"id":"cvb4c59cvrlv","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"partial_outage"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"degraded_performance","new_status":"partial_outage"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We've seen slowness in processing of Tonkean items. We're investigating the root cause and a possible fix.","created_at":"2026-04-15T10:32:02.970-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-15T10:32:02.970-07:00","display_at":"2026-04-15T10:32:02.970-07:00","deliver_notifications":true,"tweet_id":null,"id":"4zy7z0z4m8m9","incident_id":"p2lkkvcgjv86","custom_tweet":null,"affected_components":[{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":"On April 15, 2026 at 19:00 \\(UTC\\+3\\), a configuration issue triggered a loop within an automation module, causing repeated item creation and excessive system activity. This led to a significant increase in load and degraded platform performance.\n\nCustomers experienced elevated latency, including slower interface loading, delays when progressing through processes, and slower execution of automated actions. No data loss occurred.\n\nThe issue was identified within approximately one hour. Mitigation actions included stopping the loop and clearing overloaded processing queues. Full service was restored for all customers by April 16 at 12:23 \\(UTC\\+3\\).\n\nThe root cause was a trigger configuration that continuously re-executed under certain conditions, resulting in unbounded item creation and system load.\n\nTo prevent recurrence, we are implementing safeguards including rate limits on actions and item creation, defined execution quotas at the module and solution level, and automatic halting of processes when limits are exceeded with visibility in execution history.","postmortem_body_last_updated_at":"2026-04-16T06:52:30.711-07:00","postmortem_ignored":false,"postmortem_published_at":"2026-04-16T06:52:30.711-07:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"403 Forbidden Errors Detected on Specific Endpoints – Under Investigation","status":"resolved","created_at":"2025-11-25T00:51:43.119-08:00","updated_at":"2025-11-27T02:08:27.997-08:00","monitoring_at":null,"resolved_at":"2025-11-25T01:09:59.858-08:00","impact":"major","shortlink":"https://stspg.io/w36yh5xvrcpj","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-11-25T00:51:43.112-08:00","reminder_intervals":null,"id":"qyg6qx662kzk","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-11-25T01:09:59.858-08:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-25T01:09:59.858-08:00","display_at":"2025-11-25T01:09:59.858-08:00","deliver_notifications":true,"tweet_id":null,"id":"ywr639s8hrhn","incident_id":"qyg6qx662kzk","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"partial_outage","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"partial_outage","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"partial_outage","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2025-11-25T00:51:43.275-08:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-25T00:51:43.275-08:00","display_at":"2025-11-25T00:51:43.275-08:00","deliver_notifications":true,"tweet_id":null,"id":"qyjrzgs5ff5b","incident_id":"qyg6qx662kzk","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"partial_outage"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"partial_outage"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"partial_outage"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":"**Root Cause Analysis: Partial API Service Disruption – November 25, 2025**\n\nOn November 25, 2025, at 07:48 UTC, some users began experiencing issues in parts of the Tonkean platform. These included problems with the main menu, unexpected logouts with a “Board not found” error, and failures when connecting to integrations. 09:08 UTC, the incident was resolved, and all services were restored.\n\nThe root cause was misconfigured API servers. Some of our backend systems had incorrect internal permissions. This caused specific requests routed to that server to be rejected, resulting in the observed errors. Restarting the service removed the faulty component and replaced it with a properly configured one.\n\nThis issue only affected users whose requests were routed to the problematic server, resulting in inconsistent but frustrating impacts. No data was lost, and all services were fully restored within 80 minutes.\n\nTo prevent similar problems in the future, we are adding automated checks that verify internal permissions are correctly set whenever services start. These checks will also include self-healing logic to fix misconfigurations when possible automatically.\n\nOur systems are currently stable and are being closely monitored.","postmortem_body_last_updated_at":"2025-11-27T02:08:27.980-08:00","postmortem_ignored":false,"postmortem_published_at":"2025-11-27T02:08:27.980-08:00","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"Scheduled maintenance","status":"completed","created_at":"2025-10-26T01:40:33.876-07:00","updated_at":"2025-11-02T03:00:21.626-08:00","monitoring_at":null,"resolved_at":"2025-11-02T03:00:21.605-08:00","impact":"maintenance","shortlink":"https://stspg.io/zgpc6n90ywhj","scheduled_for":"2025-11-02T01:00:00.000-08:00","scheduled_until":"2025-11-02T03:00:00.000-08:00","scheduled_remind_prior":true,"scheduled_reminded_at":"2025-11-02T01:00:21.568-07:00","impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2025-10-26T01:40:33.866-07:00","reminder_intervals":null,"id":"ytwff5znv37k","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"completed","body":"The scheduled maintenance has been completed.","created_at":"2025-11-02T03:00:21.605-08:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-02T03:00:21.605-08:00","display_at":"2025-11-02T03:00:21.605-08:00","deliver_notifications":true,"tweet_id":null,"id":"7032srsmngk4","incident_id":"ytwff5znv37k","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"under_maintenance","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"under_maintenance","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"under_maintenance","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"under_maintenance","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"under_maintenance","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2025-11-02T01:00:23.556-08:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-02T01:00:23.556-08:00","display_at":"2025-11-02T01:00:23.556-08:00","deliver_notifications":true,"tweet_id":null,"id":"r0lzsh0cx1z0","incident_id":"ytwff5znv37k","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"under_maintenance"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"under_maintenance"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"under_maintenance"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"under_maintenance"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"under_maintenance"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2025-11-02T01:00:21.407-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-11-02T01:00:21.407-07:00","display_at":"2025-11-02T01:00:21.407-07:00","deliver_notifications":true,"tweet_id":null,"id":"hdppywfn625m","incident_id":"ytwff5znv37k","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"We will be undergoing scheduled maintenance during this time.","created_at":"2025-10-26T01:40:33.964-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-26T01:40:33.964-07:00","display_at":"2025-10-26T01:40:33.964-07:00","deliver_notifications":true,"tweet_id":null,"id":"1vyxqpdkfwth","incident_id":"ytwff5znv37k","custom_tweet":null,"affected_components":[{"code":"mc6xm3bpplhj","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"vmf96knldwm1","name":"Workflows Runtime","old_status":"operational","new_status":"operational"},{"code":"nwl9s4z9wcjb","name":"Workflow Builder","old_status":"operational","new_status":"operational"},{"code":"mys5jykzmq1x","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","old_status":"operational","new_status":"operational"},{"code":"k9d1mhrdn35t","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","old_status":"operational","new_status":"operational"},{"code":"vfpbpkykmwm9","name":"Communication Integrations","old_status":"operational","new_status":"operational"},{"code":"s899cxwtkm8f","name":"Identity Provider Integrations","old_status":"operational","new_status":"operational"},{"code":"qm9h4nfp8pl8","name":"Workflow Runtime History","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Workflows Runtime","created_at":"2018-03-11T09:45:58.143-07:00","updated_at":"2026-06-25T13:15:02.814-07:00","position":1,"description":null,"showcase":true,"start_date":null,"id":"vmf96knldwm1","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interfaces (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2018-03-11T09:45:58.114-07:00","updated_at":"2026-06-25T13:15:02.828-07:00","position":2,"description":null,"showcase":true,"start_date":null,"id":"mc6xm3bpplhj","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Builder","created_at":"2022-09-08T05:39:18.830-07:00","updated_at":"2026-05-24T05:00:07.261-07:00","position":3,"description":"Issues that impact the ability of makers to build and edit business logic in modules","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"nwl9s4z9wcjb","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"User Interface Builder (Forms, Item Interfaces, Workspace Apps, Business Reports)","created_at":"2022-09-08T05:39:47.530-07:00","updated_at":"2026-05-24T05:00:07.303-07:00","position":4,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"mys5jykzmq1x","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Data Sources (Webhooks, Data Collection \u0026 Actions)","created_at":"2022-09-08T05:50:40.324-07:00","updated_at":"2026-06-25T13:15:02.841-07:00","position":5,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"k9d1mhrdn35t","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Communication Integrations","created_at":"2022-09-08T05:52:37.128-07:00","updated_at":"2026-05-24T05:00:07.363-07:00","position":6,"description":null,"showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"vfpbpkykmwm9","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Identity Provider Integrations","created_at":"2022-09-08T06:05:21.615-07:00","updated_at":"2026-05-24T05:00:07.390-07:00","position":7,"description":"Issues that impact the ability of administrators to configure and integrate authentication services with an external identity provider (such as Okta and Active Directory)","showcase":true,"start_date":"2022-09-08T00:00:00.000-07:00","id":"s899cxwtkm8f","page_id":"gvm7j1jbqtxh","group_id":null},{"status":"operational","name":"Workflow Runtime History","created_at":"2023-06-22T05:54:43.397-07:00","updated_at":"2026-05-24T05:00:07.420-07:00","position":8,"description":null,"showcase":true,"start_date":"2023-06-22T00:00:00.000-07:00","id":"qm9h4nfp8pl8","page_id":"gvm7j1jbqtxh","group_id":null}]},{"name":"AWS Regional Outage – Monitoring Impact","status":"resolved","created_at":"2025-10-20T04:35:08.650-07:00","updated_at":"2025-10-20T17:33:59.572-07:00","monitoring_at":"2025-10-20T04:35:08.689-07:00","resolved_at":"2025-10-20T17:33:59.553-07:00","impact":"none","shortlink":"https://stspg.io/tbrzzrfrzxjp","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2025-10-20T04:35:08.643-07:00","reminder_intervals":null,"id":"pt1rx9gkpt6b","page_id":"gvm7j1jbqtxh","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2025-10-20T17:33:59.553-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T17:33:59.553-07:00","display_at":"2025-10-20T17:33:59.553-07:00","deliver_notifications":false,"tweet_id":null,"id":"zz8fsgxn3k5n","incident_id":"pt1rx9gkpt6b","custom_tweet":null,"affected_components":null},{"status":"monitoring","body":"We are aware of an ongoing outage affecting AWS services, which may impact parts of our platform. Our team is actively monitoring the situation and assessing potential effects on our systems.","created_at":"2025-10-20T04:35:08.689-07:00","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2025-10-20T04:35:08.689-07:00","display_at":"2025-10-20T04:35:08.689-07:00","deliver_notifications":false,"tweet_id":null,"id":"gx5sfxqcdcgt","incident_id":"pt1rx9gkpt6b","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]}]
}