[ { "id": "ev-alerts-silent-after-metric-rename-01", "incident": "alerts-silent-after-metric-rename", "grade": "primary", "class": "repo-issue", "organisation": "OpenTelemetry", "publishedOn": "2025-07-31", "url": "https://github.com/open-telemetry/opentelemetry-collector/issues/13544", "title": "Please, stop changing the names of important prometheus metrics", "summary": "Collector releases changed the names of the collector's own Prometheus metrics three times in a few weeks, most recently in version 0.131.0, so alert rules that the documentation recommends pointed at metric names that no longer existed; a maintainer called the change unintentional and a patch release followed.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-alerts-silent-after-metric-rename-03", "incident": "alerts-silent-after-metric-rename", "grade": "practitioner", "class": "engineering-blog", "organisation": "Cloudflare", "publishedOn": "2022-05-19", "url": "https://blog.cloudflare.com/monitoring-our-monitoring/", "title": "Monitoring our monitoring: how we validate our Prometheus alert rules", "summary": "Cloudflare describes alerting rules that were left referencing renamed or removed metrics after exporter changes such as a fleet-wide node_exporter upgrade, and added label changes that made alert queries return nothing, and the linter it built to check rules against live Prometheus servers.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-alerts-silent-after-metric-rename-04", "incident": "alerts-silent-after-metric-rename", "grade": "primary", "class": "repo-issue", "organisation": "Jaeger", "publishedOn": "2026-05-04", "url": "https://github.com/jaegertracing/jaeger/issues/8479", "title": "[Bug]: Prometheus alert rules in monitoring mixin use v1 metrics that don't exist in Jaeger v2", "summary": "Jaeger v2 emits otelcol_* metrics instead of the Jaeger v1 metric names, but 6 of 7 alert rules in the project's monitoring mixin still referenced the v1 names, so alerts such as JaegerCollectorDroppingSpans could never fire; the merged fix removed the obsolete v1 rules.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-all-sessions-invalidated-01", "incident": "all-sessions-invalidated", "grade": "primary", "class": "status-page", "organisation": "Mews", "publishedOn": "2025-04-11", "url": "https://status.mews.com/incidents/dz9w5r8ysrbh", "title": "Login issue on Mews POS app", "summary": "A hotfix for a TLS issue changed a secret key during a configuration clean-up, which invalidated every logged-in session on Android and web admin and forced all users to log in again.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-all-sessions-invalidated-02", "incident": "all-sessions-invalidated", "grade": "primary", "class": "status-page", "organisation": "Phrase", "publishedOn": "2026-02-23", "url": "https://status.phrase.com/incidents/b9s7rx3ywbjj", "title": "Performance Disruption of Phrase TMS CAT Web Editor and Project Management (EU) on January 29, 2026 between 4:56 PM CET and 5:46 PM CET", "summary": "A deployment migrated session storage to a new caching solution while application instances were updated at different times, so instances used different session stores and UI users in the EU region were repeatedly logged out.", "impactClass": "availability", "timeToDetectMinutes": 26, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-all-sessions-invalidated-03", "incident": "all-sessions-invalidated", "grade": "primary", "class": "status-page", "organisation": "Organization for Transformative Works", "publishedOn": "2024-10-07", "url": "https://www.otwstatus.org/incidents/1gm83clkg5wm", "title": "AO3 elevated \"Session Expired\" errors", "summary": "A new Archive of Our Own release changed session cookies to an encrypted format and switched cookie digests from SHA1 to SHA256 as part of a Rails update, which during a rolling deployment produced Session Expired errors and forced logouts.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-all-sessions-invalidated-05", "incident": "all-sessions-invalidated", "grade": "practitioner", "class": "engineering-blog", "organisation": "Can Olcer", "publishedOn": "2021-10-29", "url": "https://dev.to/canolcer/rails-prevent-users-from-logging-out-after-each-deployment-3ag9", "title": "Rails: Prevent users from logging out after each deployment", "summary": "A developer describes a Rails app that logged out all users after every deployment because the hosting platform set a new secret_key_base on each deploy, so existing session cookies could not be decrypted.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-analytics-event-schema-broken-01", "incident": "analytics-event-schema-broken", "grade": "primary", "class": "status-page", "organisation": "Fullstory", "publishedOn": "2022-08-02", "url": "https://status.fullstory.com/incidents/bc7yhtrdysg5", "title": "FS.event properties dropped", "summary": "A release of the fs.js web capture script dropped all properties from logged Custom Events for about three days, so segments, metrics, funnels, conversions and data exports relying on those properties were affected.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": "integration", "verifiedOn": "2026-10-21" }, { "id": "ev-analytics-event-schema-broken-02", "incident": "analytics-event-schema-broken", "grade": "primary", "class": "status-page", "organisation": "Amplitude", "publishedOn": "2025-06-05", "url": "https://status.amplitude.com/incidents/cnln58d753js", "title": "Unexpected \"[Segment]\" Prefix in Event Types", "summary": "For about five and a half hours, events sent from Segment through the Amplitude Actions destination arrived with a [Segment] prefix on the event name, so incorrect event names appeared in Amplitude and charts and workflows relying on existing names were disrupted until Segment deployed a fix and Amplitude merged the data.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-api-key-rotation-breaks-integrations-01", "incident": "api-key-rotation-breaks-integrations", "grade": "primary", "class": "status-page", "organisation": "Atlassian Statuspage", "publishedOn": "2026-03-10", "url": "https://metastatuspage.com/incidents/ctw5xv6t643h", "title": "Statuspage API Access Issues", "summary": "A planned cleanup of old Statuspage API keys also disabled new keys, and customers' API-based integrations failed with authorization errors until keys were restored from a database backup.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-api-key-rotation-breaks-integrations-03", "incident": "api-key-rotation-breaks-integrations", "grade": "primary", "class": "own-postmortem", "organisation": "Microsoft", "publishedOn": "2021-03-16", "url": "https://learn.microsoft.com/en-us/answers/questions/315440/notification-authentication-errors-across-multiple", "title": "Notification: Authentication errors across multiple Microsoft services - Validating Mitigation", "summary": "An automated Azure AD key rotation removed a signing key that had been marked to retain, and Microsoft and third-party applications that relied on the published key metadata stopped trusting tokens signed with it.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-api-version-dropped-while-old-app-live-01", "incident": "api-version-dropped-while-old-app-live", "grade": "primary", "class": "status-page", "organisation": "MongoDB", "publishedOn": "2026-03-02", "url": "https://status.mongodb.com/incidents/rtbthsrxzt6x", "title": "MongoDB Atlas Admin API: Temporary failures for tools using older API versions", "summary": "MongoDB's planned retirement of older versions of its cloud database Admin API made installed tools still pinned to those versions, including the Terraform provider and the vendor command-line tool, fail with HTTP 410 for about 36 hours until the end-of-life date was extended through a configuration change.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-api-version-dropped-while-old-app-live-03", "incident": "api-version-dropped-while-old-app-live", "grade": "primary", "class": "status-page", "organisation": "App Center", "publishedOn": "2020-12-09", "url": "https://status.appcenter.ms/incidents/7930rkfgtg1y", "title": "Temporarily Restoring Old Versions of App Center CLI and Release Upload APIs", "summary": "App Center deprecated its Release Upload APIs as part of the HockeyApp shutdown, which broke old versions of the App Center CLI and other clients that still called those APIs, and the team temporarily restored the old APIs until 1 February 2021.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-app-store-rollback-not-possible-01", "incident": "app-store-rollback-not-possible", "grade": "primary", "class": "status-page", "organisation": "Twilio (Authy)", "publishedOn": "2024-10-16", "url": "https://status.twilio.com/incidents/hjmj47mkbry6", "title": "Latest Version of the Authy App for iOS Is Crashing", "summary": "The latest Authy iOS release crashed on open for some users; the fixed version 26.2.2 had to pass App Store review before release, and users recovered only after upgrading to it.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-app-store-rollback-not-possible-02", "incident": "app-store-rollback-not-possible", "grade": "primary", "class": "status-page", "organisation": "Spendesk", "publishedOn": "2026-09-21", "url": "https://status.spendesk.com/incidents/4gndfyrx7ndj", "title": "Login : iOS Email Links Not Completing Sign-In", "summary": "A Spendesk iOS app release stopped email login links from completing sign-in for some users; the team halted the rollout and waited for Apple to review a hotfix, which was available about two days after the first report.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-app-store-rollback-not-possible-03", "incident": "app-store-rollback-not-possible", "grade": "primary", "class": "status-page", "organisation": "Fleetio", "publishedOn": "2022-07-25", "url": "https://status.fleetio.com/incidents/b33pzwt141m0", "title": "Fleetio Go 4.8.3 on iOS users experiencing unexpected behavior when completing inspections", "summary": "Inspection items froze in Fleetio Go version 4.8.3 on iOS; the fix went through QA and Apple approval before release, and users were directed to the website in the meantime.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-app-store-rollback-not-possible-04", "incident": "app-store-rollback-not-possible", "grade": "primary", "class": "status-page", "organisation": "Seesaw", "publishedOn": "2024-11-07", "url": "https://status.seesaw.me/incidents/4mrs2fwywwk6", "title": "iOS 10.50.0 - Camera Roll Photos Not Displaying", "summary": "Camera roll photos stopped displaying in Creative Tools in Seesaw iOS app 10.50.0; the fix waited for Apple approval and shipped in 10.51.1 while workarounds were sent to customers.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-app-store-rollback-not-possible-05", "incident": "app-store-rollback-not-possible", "grade": "primary", "class": "status-page", "organisation": "Lightspeed", "publishedOn": "2026-06-03", "url": "https://status.lightspeedhq.com/incidents/d69yp9rwsltj", "title": "K-series: Legacy Split payment issue with app version 26.22 only", "summary": "In Lightspeed K-Series app version 26.22, orders using legacy split payments closed without payment confirmation; the fix waited to become available on the App Store and shipped as hotfix 26.22.1 the next day.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-asset-cache-serves-old-bundle-04", "incident": "asset-cache-serves-old-bundle", "grade": "primary", "class": "status-page", "organisation": "DrChrono", "publishedOn": "2022-03-03", "url": "https://status.drchrono.com/incidents/lgsk9k9ckbx0", "title": "Offices not appearing on the calendar page", "summary": "After a DrChrono deployment, some JavaScript file URLs lacked a cache-busting hash, so browsers mixed cached JavaScript files from the previous deployment with files from the current one, errors were thrown and UI widgets such as the calendar office list did not appear until a hard refresh.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-asset-cache-serves-old-bundle-05", "incident": "asset-cache-serves-old-bundle", "grade": "primary", "class": "status-page", "organisation": "UiPath", "publishedOn": "2026-03-25", "url": "https://status.uipath.com/incidents/cz06zycl03bp", "title": "Multiple Regions - Agents - Agent Builder might not load for some customers.", "summary": "Following a UiPath deployment, browsers kept serving a previously cached remoteEntry.js that referenced assets no longer on the server, so the Agent Builder frontend failed to load with 404 errors for customers in 73 organisations until caches expired or were cleared.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-asset-cache-serves-old-bundle-06", "incident": "asset-cache-serves-old-bundle", "grade": "primary", "class": "status-page", "organisation": "Hospitable", "publishedOn": "2026-08-31", "url": "https://status.hospitable.com/incident/1043640", "title": "Some users reporting the inability to log in to the application", "summary": "After a Hospitable deploy, a stale JavaScript chunk cached with a success response at the CDN edge kept being served to some browsers, which loaded a broken bundle and could not complete login until the CDN was purged and frontend caching was removed.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-autoscaling-policy-misconfigured-01", "incident": "autoscaling-policy-misconfigured", "grade": "primary", "class": "status-page", "organisation": "Onfido", "publishedOn": "2021-06-21", "url": "https://status.onfido.com/incidents/lxg40fqbypgw", "title": "API Elevated Error Rate", "summary": "An Onfido API release changed the auto-scaling rule to make it more reactive, but the change reduced serving instances to their off-hours minimum baseline, causing increased latency and timeouts on api.onfido.com in the EU from 13:02 to 13:25 GMT until the release was rolled back.", "impactClass": "availability", "timeToDetectMinutes": 9, "testClassNamed": "load", "verifiedOn": "2026-10-21" }, { "id": "ev-autoscaling-policy-misconfigured-02", "incident": "autoscaling-policy-misconfigured", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2026-07-13", "url": "https://www.githubstatus.com/incidents/q27ttsnp0x4g", "title": "Actions runs are experiencing failures to start", "summary": "A configuration change to an internal GitHub autoscaling component carried outdated capacity threshold values and scaled a GitHub Actions service below its required baseline, so up to 30 percent of Actions jobs failed to start between 13:11 and 13:53 UTC until the change was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-autoscaling-policy-misconfigured-05", "incident": "autoscaling-policy-misconfigured", "grade": "primary", "class": "status-page", "organisation": "Atlassian", "publishedOn": "2022-03-14", "url": "https://status.trello.com/incidents/sgh376qp9ww4", "title": "Login failures", "summary": "Atlassian lowered the provisioned read and write capacity values of a newly introduced DynamoDB based configuration data store whose autoscaling was misconfigured, so the store did not scale up under traffic and logins to Trello, Statuspage and other Atlassian sites failed for about 42 minutes until use of the new store was disabled.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-autoscaling-policy-misconfigured-06", "incident": "autoscaling-policy-misconfigured", "grade": "primary", "class": "status-page", "organisation": "LiveKit", "publishedOn": "2025-08-22", "url": "https://status.livekit.io/incidents/8pcxfyvj4ymb", "title": "Temporary Egress Availability Issues", "summary": "A change affecting LiveKit's autoscaling metrics caused its canary clusters to stop scaling, and about 2 percent of StartEgress requests failed with 503 errors from 15:58 to 16:18 UTC until the change was reverted.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-autoscaling-policy-misconfigured-07", "incident": "autoscaling-policy-misconfigured", "grade": "primary", "class": "status-page", "organisation": "HubSpot", "publishedOn": "2025-09-26", "url": "https://status.hubspot.com/incidents/0pt3l1dvjrv9", "title": "HubSpot Application in North America was unavailable for some users", "summary": "A HubSpot deployment created auto-scaling issues that overloaded services, and some customers in North America could not use the HubSpot application between 12:17 PM and 12:28 PM EDT on September 26, 2025.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-background-job-queue-backlog-01", "incident": "background-job-queue-backlog", "grade": "primary", "class": "status-page", "organisation": "Heroku", "publishedOn": "2022-09-23", "url": "https://status.heroku.com/api/v4/incidents/2451", "title": "Degraded availability", "summary": "A change to the application that manages Common Runtime infrastructure included a dependency upgrade that greatly reduced task worker throughput; the work queue built up, new router instances were not initialised correctly and were deleted, and requests to dynos failed until the change was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-background-job-queue-backlog-02", "incident": "background-job-queue-backlog", "grade": "primary", "class": "status-page", "organisation": "Buildkite", "publishedOn": "2026-08-10", "url": "https://www.buildkitestatus.com/incidents/vtz6bhz26gj2", "title": "Latency on Pipelines", "summary": "A routine Rails update pulled a new major version of connection_pool that Sidekiq could not use; internal Sidekiq scheduler threads stopped on startup, so recurring background work stopped and queued jobs waited longer for dispatch until the library was pinned back.", "impactClass": "degraded", "timeToDetectMinutes": 15, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-background-job-queue-backlog-05", "incident": "background-job-queue-backlog", "grade": "primary", "class": "status-page", "organisation": "PyPI", "publishedOn": "2022-06-28", "url": "https://status.python.org/incidents/l16ylxktn4r9", "title": "Queue delay affecting CDN purges and emails", "summary": "A deployed periodic task for 2FA statistics ran a slow row-counting query every 5 minutes; over several days these tasks occupied more of the workers until the central task queue held nearly 50K pending tasks and CDN purges and emails were delayed by more than 14 hours.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-cache-invalidation-storm-01", "incident": "cache-invalidation-storm", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2026-04-08", "url": "https://github.blog/news-insights/company-news/github-availability-report-march-2026/", "title": "GitHub availability report: March 2026", "summary": "A GitHub deployment meant to reduce writes to the user settings cache contained a bug that expired every user's cache at once; recalculating and rewriting the entries caused replication delays across dependent services until the deployment was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-cache-invalidation-storm-03", "incident": "cache-invalidation-storm", "grade": "primary", "class": "own-postmortem", "organisation": "Slack", "publishedOn": "2022-04-26", "url": "https://slack.engineering/slacks-incident-on-2-22-22/", "title": "Slack's Incident on 2-22-22", "summary": "During a percentage-based rollout of a Consul agent upgrade at Slack, agent restarts on memcached hosts caused cache nodes to be replaced or flushed, the cache hit rate dropped and scatter queries for group DM membership overloaded a Vitess keyspace, so many users could not connect.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-certificate-expired-01", "incident": "certificate-expired", "grade": "primary", "class": "status-page", "organisation": "Jamf", "publishedOn": "2025-06-16", "url": "https://status.jamf.com/incidents/l232fkg3j2r8", "title": "Jamf Security Cloud - secure DNS services are not working due to a wrong certificate", "summary": "An SSL certificate renewed in May 2025 was only partly distributed because the deployment process had an error, so when the old certificate expired on 15 June 2025 devices using Jamf SecureDNS lost internet access for 270 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": "synthetic", "verifiedOn": "2026-10-21" }, { "id": "ev-certificate-expired-02", "incident": "certificate-expired", "grade": "primary", "class": "status-page", "organisation": "Lucidworks", "publishedOn": "2026-07-28", "url": "https://status.lucidworks.com/incidents/vxmrbxrlzbcl", "title": "SaaS Platform User Interface and Signals Beacon Disruption", "summary": "A manually renewed shared TLS certificate for the Lucidworks SaaS Platform domains expired without a replacement after the advance expiry alerts stopped reaching the responsible team, so the platform UI returned certificate errors and Beacon signal ingestion stopped for about 65 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": "synthetic", "verifiedOn": "2026-10-21" }, { "id": "ev-certificate-expired-03", "incident": "certificate-expired", "grade": "primary", "class": "status-page", "organisation": "JumpCloud", "publishedOn": "2025-10-08", "url": "https://status.jumpcloud.com/incidents/5jfkd0gh033r", "title": "LDAP Degredation", "summary": "An internal TLS certificate expired after the automated renewal job failed to run, so dependent LDAP services could not renew their own certificates and some customers saw intermittent LDAP authentication failures.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-certificate-expired-05", "incident": "certificate-expired", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2025-12-11", "url": "https://github.blog/news-insights/company-news/github-availability-report-november-2025/", "title": "GitHub Availability Report: November 2025", "summary": "An expired TLS certificate used for internal service-to-service communication made all Git operations over SSH and HTTP, and raw file access, fail on GitHub for 64 minutes on 18 November 2025.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-checkout-blocked-for-segment-01", "incident": "checkout-blocked-for-segment", "grade": "primary", "class": "status-page", "organisation": "Chargebee", "publishedOn": "2024-08-16", "url": "https://status.chargebee.com/incidents/kh0dmwpcbl9g", "title": "Payment Processing Issues With Adyen Gateway", "summary": "Chargebee reported that after it moved its Adyen integration from the Classic to the Checkout integration on 15 August 2024, customer transactions through Adyen that used the temporary (CSE) token workflow failed with 4xx errors until the change was reverted.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-checkout-blocked-for-segment-02", "incident": "checkout-blocked-for-segment", "grade": "primary", "class": "repo-issue", "organisation": "WooCommerce", "publishedOn": "2026-10-09", "url": "https://github.com/woocommerce/woocommerce/issues/69643", "title": "Checkout block rejects legacy 4-digit Argentine postcodes (e.g. 1213) since 11.2.0", "summary": "The WooCommerce issue reports that from release 11.2.0 the Checkout block rejected four-digit Argentine postcodes after a validation package bump, so shoppers in Argentina could not place orders and merchants rolled back to 11.1.x.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-cold-start-latency-regression-02", "incident": "cold-start-latency-regression", "grade": "primary", "class": "repo-issue", "organisation": "Microsoft (.NET runtime)", "publishedOn": "2025-10-01", "url": "https://github.com/dotnet/runtime/issues/120288", "title": "Detecting .NET 10 Regression in startup performance affecting AWS Lambda", "summary": "An AWS Lambda engineer measured a 5 to 10 percent slower first invoke on .NET 10 than on .NET 8 in Lambda; .NET team members attributed it to extra type loading cost from types added in .NET 10 and backported a lazy initialisation fix to .NET 10 servicing.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-cold-start-latency-regression-03", "incident": "cold-start-latency-regression", "grade": "primary", "class": "repo-issue", "organisation": "Amazon Web Services (AWS SDK for JavaScript v3)", "publishedOn": "2025-03-02", "url": "https://github.com/aws/aws-sdk-js-v3/issues/6914", "title": "Lambda Node 22 coldstart latency regression - defer loading of node:http", "summary": "After moving Lambda functions that use the AWS SDK for JavaScript v3 from Node 20 to Node 22, init duration rose by about 50 ms because the SDK eagerly imported node:http; the SDK team shipped lazy loading in @smithy/node-http-handler 4.5.0.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-config-change-without-restart-semantics-01", "incident": "config-change-without-restart-semantics", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2022-11-02", "url": "https://github.blog/news-insights/company-news/github-availability-report-october-2022/", "title": "GitHub Availability Report: October 2022", "summary": "After a Codespaces secret rotation with a missed step, some downstream components kept using the old secret because they only loaded the new one on process restart, and one service instance in West Europe was not rotated at all, so port forwarding and later codespace creation failed.", "impactClass": "availability", "timeToDetectMinutes": 247, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-config-change-without-restart-semantics-02", "incident": "config-change-without-restart-semantics", "grade": "primary", "class": "own-postmortem", "organisation": "Cloudflare", "publishedOn": "2025-11-18", "url": "https://blog.cloudflare.com/18-november-2025-outage/", "title": "Cloudflare outage on November 18, 2025", "summary": "A database permissions change was rolled out gradually across a ClickHouse cluster, so the Bot Management feature file was generated correctly on some nodes and wrongly on updated nodes, and the network alternated between good and bad files until every node produced the oversized file.", "impactClass": "availability", "timeToDetectMinutes": 26, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-connection-pool-exhausted-01", "incident": "connection-pool-exhausted", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2025-02-28", "url": "https://www.githubstatus.com/incidents/36ftd36c921f", "title": "Elevated Request Latency for Write operations on github.com and api.github.com", "summary": "A newly deployed background job at GitHub increased load on the primary database hosts and exhausted the connection pool, raising write latency and request timeouts until the job was halted.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-connection-pool-exhausted-02", "incident": "connection-pool-exhausted", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2026-06-08", "url": "https://www.githubstatus.com/incidents/tm84vy58jq0h", "title": "Elevated rate of Git push errors", "summary": "A recently enabled code path at GitHub issued an expensive query against a primary database, exhausting its connection pool so that an average of 31 percent of Git pushes failed for nine minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-connection-pool-exhausted-05", "incident": "connection-pool-exhausted", "grade": "primary", "class": "own-postmortem", "organisation": "GitLab", "publishedOn": "2024-11-05", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/18817", "title": "Incident Review: The sidekiq_queueing SLI of the sidekiq service on shard low-urgency-cpu-bound has an apdex violating SLO", "summary": "GitLab rolled a feature flag out to 100 percent of pods that made the database load balancer check out an extra connection for health checks during every Sidekiq job, which saturated the urgent Sidekiq pgbouncer pool until the flag was disabled.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-connection-pool-exhausted-06", "incident": "connection-pool-exhausted", "grade": "primary", "class": "status-page", "organisation": "tada", "publishedOn": "2023-01-11", "url": "https://status.usetada.com/incident/152392", "title": "Connection issue to database", "summary": "After a scheduled release at tada, a code condition opened database connections without closing them, and when real traffic arrived new database connections timed out, taking down the Integration API and Wallet API until the code was fixed and redeployed.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-connection-pool-exhausted-07", "incident": "connection-pool-exhausted", "grade": "primary", "class": "status-page", "organisation": "Close", "publishedOn": "2020-11-25", "url": "https://status.close.com/incidents/1r0x2kk00s48", "title": "Application loading issues", "summary": "A code change deployed at Close left database connections open where call paths crossed into older session management code, and stale connections accumulated until the application exhausted all connections to a backend database and all Close systems were disrupted for 11 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-container-image-base-change-01", "incident": "container-image-base-change", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2023-12-15", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/17283", "title": "2023-12-15: docker:24.0.7-dind failing to start on GitLab Runner (saas)", "summary": "The docker 24.0.7 image release updated its alpine base, which switched iptables to nf_tables; GitLab's SaaS runner infrastructure did not support nf_tables, so docker:24.0.7-dind failed to start and CI jobs could not build images until the upstream image was fixed.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-container-image-base-change-02", "incident": "container-image-base-change", "grade": "primary", "class": "own-postmortem", "organisation": "CircleCI", "publishedOn": "2018-12-08", "url": "https://discuss.circleci.com/t/convenience-images-update-2018-11-27-node-variants-incident/27025", "title": "Convenience Images Update: 2018.11.27 Node variants incident", "summary": "Docker moved the official Node images' carbon tag from a Debian Jessie base to a Debian Stretch base, CircleCI's image build automation then left an extra FROM statement in Node variants of its convenience images, and the rebuilt images were missing the circleci user and many failed to start in CircleCI jobs.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-container-image-base-change-03", "incident": "container-image-base-change", "grade": "primary", "class": "repo-issue", "organisation": "Crafty Controller", "publishedOn": "2025-01-20", "url": "https://gitlab.com/crafty-controller/crafty-4/-/issues/521", "title": "Docker Image 24.04 rebase - non-root user issue. (#521)", "summary": "Crafty Controller changed its Docker base image from ubuntu:22.04 to ubuntu:24.04 in release 4.4.5; the new image ships a default ubuntu user with UID 1000 that conflicted with the project's own non-root user, so existing containers with mounted volumes lost file access and entered a boot loop in versions 4.4.5 and 4.4.6 until 4.4.7 removed the shipped user.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-cors-policy-blocks-frontend-01", "incident": "cors-policy-blocks-frontend", "grade": "primary", "class": "status-page", "organisation": "TrustPoint", "publishedOn": "2026-05-06", "url": "https://status.trustpoint.ai/cmou1i25p02hpi2ixk7f12b36", "title": "Website is down", "summary": "A TrustPoint backend deployment changed CORS headers, which stopped the frontend from communicating with the API until the CORS configuration was corrected and redeployed.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-cors-policy-blocks-frontend-02", "incident": "cors-policy-blocks-frontend", "grade": "primary", "class": "status-page", "organisation": "Stoplight", "publishedOn": "2023-06-09", "url": "https://stoplight.status.smartbear.com/incidents/x8t8mdrlv9l4", "title": "Login issue for workspaces using custom domains - Use *.stoplight.io domain for workspace login", "summary": "A Stoplight production release caused a CORS error that blocked login for workspaces accessed through custom domains, and a release rollback resolved it.", "impactClass": "availability", "timeToDetectMinutes": 139, "testClassNamed": "e2e", "verifiedOn": "2026-10-21" }, { "id": "ev-cors-policy-blocks-frontend-03", "incident": "cors-policy-blocks-frontend", "grade": "primary", "class": "status-page", "organisation": "ChartHop", "publishedOn": "2022-11-09", "url": "https://status.charthop.com/incidents/rv07bqm6144z", "title": "UI failing to load", "summary": "An upgraded third-party JavaScript component began sending a baggage header that ChartHop's cross-origin allowed headers policy did not accept, so browsers rejected API requests from the UI.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-crash-on-older-os-version-01", "incident": "crash-on-older-os-version", "grade": "primary", "class": "status-page", "organisation": "Patreon", "publishedOn": "2026-02-19", "url": "https://status.patreon.com/incidents/pg6n0dc64g4s", "title": "App launch failure on iOS 16", "summary": "Patreon iOS app version 126.8.0 did not launch for users on iOS 16 until a fixed version 126.8.1 was released the next day.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-crash-on-older-os-version-02", "incident": "crash-on-older-os-version", "grade": "primary", "class": "repo-issue", "organisation": "Tuta", "publishedOn": "2026-05-01", "url": "https://github.com/tutao/tutanota/issues/10755", "title": "Version 346.260428.0 crashes on iOS/iPad OS 17 and older", "summary": "After the Tuta Mail iOS update to version 346.260428.0 the app crashed on launch on iOS and iPadOS 17 and older; a maintainer attributed it to a bug in Apple's build tooling and directed users to the browser until a fixed build shipped.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-critical-path-broken-in-one-browser-03", "incident": "critical-path-broken-in-one-browser", "grade": "primary", "class": "repo-issue", "organisation": "Nextcloud", "publishedOn": "2025-03-16", "url": "https://github.com/nextcloud/server/issues/51506", "title": "[Bug]: safari on ios16 can't access web ui after update to 31.0.1 from 31.0.0, downgrade will resume ok", "summary": "After Nextcloud Server was updated from 31.0.0 to 31.0.1, the web interface no longer loaded in Safari on iOS versions below 16.4 because a frontend dependency used a regular expression those versions do not support.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-critical-path-broken-in-one-browser-04", "incident": "critical-path-broken-in-one-browser", "grade": "primary", "class": "status-page", "organisation": "Files.com", "publishedOn": "2022-09-06", "url": "https://status.files.com/incidents/4q2yfg9l8rw3", "title": "Issue Accessing Files Page When Connecting with Safari", "summary": "A Files.com deployment included a software bug that affected only the Safari browser, so the Files page of the web interface did not load in Safari while other browsers worked, until the deployment was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-csp-blocks-required-script-01", "incident": "csp-blocks-required-script", "grade": "primary", "class": "repo-issue", "organisation": "FileBrowser Quantum", "publishedOn": "2026-09-23", "url": "https://github.com/gtsteffaniak/filebrowser/issues/2995", "title": "[Bug] [Critical] Intermittent blank page in v2.0.8-beta: CSP nonce generated with base64.StdEncoding is silently stripped by html/template", "summary": "After a deployment was upgraded from FileBrowser Quantum v2.0.0-beta to v2.0.8-beta, which enables a nonce-based Content Security Policy, the nonce attribute was often dropped from the inline script tag, so the browser blocked the script and the single-page app showed a blank page.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-csp-blocks-required-script-02", "incident": "csp-blocks-required-script", "grade": "primary", "class": "repo-issue", "organisation": "Apache Superset", "publishedOn": "2023-07-05", "url": "https://github.com/apache/superset/issues/24597", "title": "OAuth login broken with Content Security Policy", "summary": "After moving to the latest Apache Superset docker image, which added nonce attributes and a strict script-src Content Security Policy, the browser refused an inline event handler on the login page and the OAuth login button did nothing.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-deep-link-broken-after-routing-change-02", "incident": "deep-link-broken-after-routing-change", "grade": "primary", "class": "status-page", "organisation": "Branch", "publishedOn": "2022-06-22", "url": "https://status.branch.io/incidents/nvcfm5j7bwl2", "title": "Link Redirection Issue", "summary": "A configuration change at the Branch deep linking service made link redirection fail for 22 minutes, sending users to the app's fallback domain or to an error page instead of the linked destination.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-deep-link-broken-after-routing-change-03", "incident": "deep-link-broken-after-routing-change", "grade": "primary", "class": "own-postmortem", "organisation": "Yivi (Caesar Groep)", "publishedOn": "2025-07-01", "url": "https://docs.yivi.app/blog/2025-juni-post-mortem", "title": "Post-mortem of the June 29th and 30th 2025 incident", "summary": "A domain migration of irma.app made it redirect traffic to yivi.app, and because iOS does not follow redirects for Universal Links, iOS users who installed the Yivi app after the migration could not open the app through Universal Links until the domain was pointed back to its original server.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-dependency-removed-from-registry-01", "incident": "dependency-removed-from-registry", "grade": "primary", "class": "own-postmortem", "organisation": "npm, Inc.", "publishedOn": "2016-03-23", "url": "https://blog.npmjs.org/post/141577284765/kik-left-pad-and-npm", "title": "kik, left-pad, and npm", "summary": "An author unpublished 273 packages from the npm registry, including left-pad, and projects that depended on it directly or through dependency chains such as babel and atom failed to install for about 2.5 hours until npm re-published the original version from a backup.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-dependency-removed-from-registry-02", "incident": "dependency-removed-from-registry", "grade": "primary", "class": "repo-issue", "organisation": "Rails", "publishedOn": "2021-03-24", "url": "https://github.com/rails/rails/issues/41750", "title": "Dependency on mimemagic 0.3.x no longer valid", "summary": "The mimemagic gem yanked all builds prior to 0.4.0 to resolve a licensing issue, and because Rails activestorage depended on mimemagic ~> 0.3.2, bundles locked to mimemagic 0.3.5 could not install Rails until the maintainer released a compatible 0.3.6.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-dependency-removed-from-registry-06", "incident": "dependency-removed-from-registry", "grade": "primary", "class": "repo-issue", "organisation": "Sylva project", "publishedOn": "2025-09-18", "url": "https://gitlab.com/sylva-projects/sylva-core/-/issues/2889", "title": "failures due to removal of some docker.io/bitnami/... images (in parrticular kyverno w. bitnami/kubectl image)", "summary": "After the docker.io/bitnami/kubectl image was removed from Docker Hub, Sylva CI pipelines failed because a Kyverno cleanup job could not pull bitnami/kubectl:1.30.2, an image reference missed when the project switched its images to docker.io/bitnamilegacy.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-disk-full-from-new-log-volume-01", "incident": "disk-full-from-new-log-volume", "grade": "primary", "class": "status-page", "organisation": "Benevity", "publishedOn": "2024-09-18", "url": "https://status.benevity.org/incidents/0x9f9crj5drm", "title": "Benevity Status - Spark unavailable", "summary": "An update to Benevity Spark meant to enhance system monitoring generated a larger volume of log files than anticipated, servers reported high space utilization, and users saw intermittent 500 error pages for about 38 minutes.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-disk-full-from-new-log-volume-04", "incident": "disk-full-from-new-log-volume", "grade": "primary", "class": "status-page", "organisation": "SCORM Cloud", "publishedOn": "2021-09-24", "url": "https://status.cloud.scorm.com/incidents/ckc94twltqbd", "title": "SCORM Cloud Status - Slow Response Times", "summary": "An operating system update deployed to SCORM Cloud application servers produced far more Linux kernel audit logs, and with insufficient log rotation the root partitions filled until the disk health check removed many servers at once during morning traffic, causing timeouts and partial outages.", "impactClass": "availability", "timeToDetectMinutes": 1366, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-disk-full-from-new-log-volume-05", "incident": "disk-full-from-new-log-volume", "grade": "primary", "class": "repo-issue", "organisation": "OPNsense", "publishedOn": "2026-01-16", "url": "https://github.com/opnsense/hostwatch/issues/8", "title": "ERROR hostwatch: Error reading from capture: libpcap error: The interface disappeared · Issue #8 · opnsense/hostwatch", "summary": "After an OPNsense update that added the new hostwatch automatic discovery service, hostwatch logged the same libpcap error several times per second when virtual interfaces such as WireGuard were destroyed and recreated, filling firewall disks and upstream syslog hosts until maintainers shipped hostwatch 1.0.5, which disabled two offending log messages.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-dns-change-propagation-outage-01", "incident": "dns-change-propagation-outage", "grade": "primary", "class": "own-postmortem", "organisation": "Rust Infrastructure Team", "publishedOn": "2023-02-08", "url": "https://blog.rust-lang.org/inside-rust/2023/02/08/dns-outage-portmortem/", "title": "DNS Outage on 2023-01-25", "summary": "A Terraform deployment to crates.io production removed the existing TLS certificate and DNS records before issuing a new certificate, leaving static.crates.io with no DNS record for about 10 to 15 minutes until Terraform recreated the records.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-dns-change-propagation-outage-02", "incident": "dns-change-propagation-outage", "grade": "primary", "class": "own-postmortem", "organisation": "Slack", "publishedOn": "2021-11-29", "url": "https://slack.engineering/what-happened-during-slacks-dnssec-rollout/", "title": "The Case of the Recursive Resolvers: What Happened During Slack's DNSSEC Rollout", "summary": "Slack enabled DNSSEC signing on slack.com and published the DS record; users began to see name resolution errors, and after the change was rolled back, validating resolvers that had cached the DS record returned SERVFAIL for slack.com lookups, affecting less than 1 percent of online users for about 24 hours.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-dns-change-propagation-outage-04", "incident": "dns-change-propagation-outage", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2026-03-05", "url": "https://www.githubstatus.com/incidents/g5gnt5l5hf56", "title": "Multiple services are affected, service degradation", "summary": "Redis infrastructure updates rolled out to production introduced incorrect configuration into the Redis load balancer, routing GitHub Actions internal traffic to an incorrect host, so between 16:24 and 19:30 UTC on March 5, 2026, 95 percent of workflow runs failed to start within 5 minutes and 10 percent failed with an infrastructure error.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-double-charge-after-deploy-04", "incident": "double-charge-after-deploy", "grade": "primary", "class": "status-page", "organisation": "Spreedly", "publishedOn": "2017-02-10", "url": "https://status.spreedly.com/incidents/9djhznmdp07p", "title": "Issues processing transactions with Ruby Gem", "summary": "On 8 February 2017 Spreedly deployed code that removed a field from some API responses; the Spreedly Ruby gem raised exceptions when parsing those responses, and customers who retried the affected transactions created duplicate charges until the change was reverted about 2.5 hours later.", "impactClass": "money", "timeToDetectMinutes": 57, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-double-charge-after-deploy-05", "incident": "double-charge-after-deploy", "grade": "primary", "class": "status-page", "organisation": "AuthVia", "publishedOn": "2025-10-15", "url": "https://status.authvia.com/incidents/3hzrsb96jhkh", "title": "Duplicate payments are possible", "summary": "On 14 October 2025 a scheduled AuthVia deployment changed logic in the transaction workflow so that some payment requests re-executed under specific timing conditions, sending duplicate payment submissions to downstream gateways until a fix was deployed.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-email-or-sms-provider-template-broken-01", "incident": "email-or-sms-provider-template-broken", "grade": "primary", "class": "repo-issue", "organisation": "Public Knowledge Project", "publishedOn": "2023-06-26", "url": "https://github.com/pkp/pkp-lib/issues/9111", "title": "Wrong variable name in PASSWORD_RESET_CONFIRM email template", "summary": "In OJS 3.4 the PASSWORD_RESET_CONFIRM email template had passwordResetUrl replaced by lostPasswordUrl, a variable that does not exist, so the password reset email went out with the variable not substituted; maintainers renamed the variable and fixed localized bodies for milestone 3.4.0-2.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-email-or-sms-provider-template-broken-08", "incident": "email-or-sms-provider-template-broken", "grade": "primary", "class": "status-page", "organisation": "Signhost Verified Signing", "publishedOn": "2021-06-14", "url": "https://status.signhost.com/incidents/79mc7z3qpr4d", "title": "Sign invite mails do not contain invite link", "summary": "After Signhost released a script version update in its email functionality, signing invitation emails were sent for about an hour without text or invite link and contained only unfilled variables, until a second rollback restored the previous version with correct mail templates.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": "integration", "verifiedOn": "2026-10-22" }, { "id": "ev-encoding-or-timezone-shift-01", "incident": "encoding-or-timezone-shift", "grade": "primary", "class": "own-postmortem", "organisation": "Sigma Computing", "publishedOn": "2023-10-06", "url": "https://community.sigmacomputing.com/t/postmortem-for-aug-10th-2023-incorrect-results-for-timezone-based-calculations/2618", "title": "Postmortem for Aug 10th, 2023: Incorrect results for timezone-based calculations", "summary": "A backend code change stopped timezones from being set in the drivers Sigma uses to connect to customer data warehouses, so users with a custom account timezone saw incorrect results for timezone-based calculations on Snowflake and later on Databricks.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-encoding-or-timezone-shift-02", "incident": "encoding-or-timezone-shift", "grade": "primary", "class": "status-page", "organisation": "ServiceChannel", "publishedOn": "2020-04-08", "url": "https://status.servicechannel.com/incidents/0kz4k7vhzqyt", "title": "ServiceChannel Status - Incorrect Work Order Timezone", "summary": "An infrastructure change to how the ServiceChannel Dashboard module is served left the timezone configuration variable set to NULL, so Work Order Notes were recorded four hours into the future and 3,178 notes needed a retroactive timezone correction.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": "config", "verifiedOn": "2026-10-22" }, { "id": "ev-env-mismatch-prod-points-to-staging-01", "incident": "env-mismatch-prod-points-to-staging", "grade": "primary", "class": "own-postmortem", "organisation": "Cloudflare", "publishedOn": "2023-11-01", "url": "https://blog.cloudflare.com/cloudflare-incident-on-october-30-2023/", "title": "Cloudflare incident on October 30, 2023", "summary": "During a progressive Workers KV release, a deployment tool bug made production reference a script deployed to the staging account, so KV returned HTTP 401 errors and dependent Cloudflare services were unavailable for 37 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-env-mismatch-prod-points-to-staging-02", "incident": "env-mismatch-prod-points-to-staging", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2024-02-26", "url": "https://www.githubstatus.com/incidents/f9ntcnd3fdvs", "title": "We are investigating reports of degraded performance.", "summary": "A misconfigured GitHub observability metric pulled in data from a newly provisioned test environment, so production monitoring reported a GitHub Packages outage that had no real customer impact.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-error-budget-burned-by-retry-storm-01", "incident": "error-budget-burned-by-retry-storm", "grade": "primary", "class": "own-postmortem", "organisation": "LaunchDarkly", "publishedOn": "2025-10-28", "url": "https://launchdarkly.com/blog/what-happened-what-we-learned-and-how-were-improving/", "title": "Our October 20 Service Disruption: What Happened, What We Learned and How We’re Improving", "summary": "During recovery from an AWS us-east-1 outage, an internal change meant to reduce load on the web application reverted flag delivery to a legacy routing path with cold caches, SDKs retried excessively, and the retry traffic overwhelmed the streaming service and its load balancer, leaving server-side SDK connections failing at about 99% globally.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-error-budget-burned-by-retry-storm-04", "incident": "error-budget-burned-by-retry-storm", "grade": "primary", "class": "own-postmortem", "organisation": "PostHog", "publishedOn": "2025-09-29", "url": "https://posthog.com/handbook/company/post-mortems/2025-09-29-flags-is-down", "title": "Feature flags service outage", "summary": "A deployment that reduced the feature flags service database connection timeout from 1 second to 300 milliseconds, during elevated writer database load, led to connection failures whose retries without circuit breakers or backoff overwhelmed the writer database and made the service unavailable.", "impactClass": "availability", "timeToDetectMinutes": 23, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-flag-default-flips-on-missing-config-03", "incident": "flag-default-flips-on-missing-config", "grade": "primary", "class": "status-page", "organisation": "Amplitude", "publishedOn": "2023-07-28", "url": "https://status.amplitude.com/incidents/ypgybsxrkn87", "title": "EU Event Segmentation Chart Formulas gated unintentionally", "summary": "An Amplitude update gated the chart formula feature, and because the EU feature flag for it was missing, EU customers fell back to the gated state for about three days.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-flag-default-flips-on-missing-config-04", "incident": "flag-default-flips-on-missing-config", "grade": "primary", "class": "own-postmortem", "organisation": "Inngest", "publishedOn": "2026-07-23", "url": "https://www.inngest.com/blog/2026-07-23-incident-rca-july-2026", "title": "Root Cause Analysis: July 2026 Service Incidents", "summary": "During an Inngest release on 10 July 2026, newly started containers could not fetch feature flag values from a third-party flag service and fell back to default configuration, which moved internal traffic onto a legacy HTTP path, overloaded the internal network and degraded function execution and run scheduling until the release was rolled back.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-flag-enabled-for-wrong-segment-01", "incident": "flag-enabled-for-wrong-segment", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2025-06-20", "url": "https://www.githubstatus.com/incidents/sd4v95zxm3np", "title": "Disruption with the GitHub mobile android application", "summary": "A GitHub App feature under internal test, controlled by a feature flag, was enforced for all GitHub-owned applications including GitHub Mobile, and the Android app could not log in new users until the flag was disabled.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-flag-enabled-for-wrong-segment-03", "incident": "flag-enabled-for-wrong-segment", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2024-07-25", "url": "https://www.githubstatus.com/incidents/q1yjht6mt3zn", "title": "Linking internal teams to external IDP groups was broken for some users between 15:17-20:44 UTC", "summary": "An experimental GitHub UI component was rolled out to customers through a feature flag by mistake, and team-to-external-group links silently failed to save until the flag was scaled back to 0 percent of actors.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-flag-enabled-for-wrong-segment-05", "incident": "flag-enabled-for-wrong-segment", "grade": "primary", "class": "own-postmortem", "organisation": "Appfarm", "publishedOn": "2026-05-13", "url": "https://community.appfarm.io/t/post-mortem-flows-feature-flag-incident-may-7-2026/2354", "title": "Post-mortem: Flows feature flag incident, May 7, 2026", "summary": "Appfarm added version-based targeting to its feature flag system for the Flows rollout, but the rule evaluated against each solution's Appfarm Create version instead of the targeted environment's version, so the flag activated in customer environments on the platform release schedule rather than on customer-initiated deploys until the flag was disabled platform-wide.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-form-submission-silently-fails-01", "incident": "form-submission-silently-fails", "grade": "primary", "class": "repo-issue", "organisation": "Hose Water", "publishedOn": "2026-09-30", "url": "https://github.com/hosewaterpw/hose-water-pressure-washing/pull/2", "title": "Fix: restore form submission delivery", "summary": "After static success pages were removed from a Next.js site hosted on Netlify, the contact and estimate form success addresses resolved to Next.js pages, so Next.js answered the POST, Netlify never captured the submissions and visitors still saw a thank-you page.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-form-submission-silently-fails-04", "incident": "form-submission-silently-fails", "grade": "primary", "class": "own-postmortem", "organisation": "Netlify", "publishedOn": "2020-10-14", "url": "https://answers.netlify.com/t/potential-missing-data-from-form-submissions-in-oct-9-14-2020/24380/1", "title": "Potential Missing data from form submissions in Oct 9-14 2020", "summary": "A Netlify code deploy on 9 October 2020 stopped the form handling service from processing updates to forms whose fields were added, removed or renamed, so visitor data submitted in those changed fields was not saved until 14 October 2020 and could not be restored.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-iam-policy-change-blocks-service-01", "incident": "iam-policy-change-blocks-service", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2026-03-11", "url": "https://github.blog/news-insights/company-news/github-availability-report-february-2026/", "title": "GitHub availability report: February 2026", "summary": "Security policies were mistakenly applied to backend storage accounts in GitHub's underlying compute provider, the policies blocked access to VM metadata, and VM operations failed so GitHub Actions hosted runners and Codespaces were unavailable in all regions until the policy changes were rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-iam-policy-change-blocks-service-03", "incident": "iam-policy-change-blocks-service", "grade": "primary", "class": "own-postmortem", "organisation": "W. W. Norton & Company", "publishedOn": "2025-11-07", "url": "https://digitalproduct.wwnorton.com/blog/postmortems/testmaker-export-outage/", "title": "Incident: Testmaker Export Outage | Norton Digital Product Guidebook", "summary": "An internal security tool enabled an AWS account-level S3 public access block, which blocked the public-read S3 uploads that Testmaker export and import services relied on until the account-level configuration was rolled back about 15 hours later.", "impactClass": "availability", "timeToDetectMinutes": 145, "testClassNamed": "synthetic", "verifiedOn": "2026-10-21" }, { "id": "ev-iam-policy-change-blocks-service-04", "incident": "iam-policy-change-blocks-service", "grade": "primary", "class": "status-page", "organisation": "Buildkite", "publishedOn": "2025-11-07", "url": "https://www.buildkitestatus.com/incidents/r9n9lqb3xrks", "title": "Buildkite Status - Increased 403 Errors on Package Registries package access", "summary": "A Terraform update to the S3 bucket policy for a bucket of customer packages overwrote the existing access policies instead of merging with them, so Debian and RubyGems package downloads returned 403 errors until the previous bucket policy was redeployed.", "impactClass": "availability", "timeToDetectMinutes": 40, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-in-app-purchase-receipt-validation-fails-01", "incident": "in-app-purchase-receipt-validation-fails", "grade": "primary", "class": "repo-issue", "organisation": "Flutter (Google)", "publishedOn": "2025-07-04", "url": "https://github.com/flutter/flutter/issues/171634", "title": "[in_app_purchase] After upgrade verificationData.serverVerificationData verified in server causes a crash", "summary": "After a minor in_app_purchase upgrade made StoreKit 2 the default on iOS, serverVerificationData changed from a base64 App Store receipt to a JWT, so unchanged server-side receipt validation code failed for new purchases; maintainers confirmed the behaviour and said it should have been a major version change.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-in-app-purchase-receipt-validation-fails-04", "incident": "in-app-purchase-receipt-validation-fails", "grade": "primary", "class": "status-page", "organisation": "RevenueCat", "publishedOn": "2024-02-19", "url": "https://status.revenuecat.com/incidents/60x0b9yt5nqs", "title": "Problem with Consumables using Google Play Store", "summary": "A change on a Google Play endpoint that RevenueCat calls to tell subscriptions from other purchases began returning an incorrect error, so RevenueCat did not process about 10 percent of Android non-subscription purchases until it deployed handling for the error and replayed the receipts.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-irreversible-migration-fails-midway-01", "incident": "irreversible-migration-fails-midway", "grade": "primary", "class": "status-page", "organisation": "Increase", "publishedOn": "2024-03-26", "url": "https://status.increase.com/incidents/f4mgxknbfjc2", "title": "API unavailability", "summary": "A database migration at Increase partially failed, leaving the database schema out of sync with the deployed code, and the API was unavailable for 24 minutes until a data migration and a new API server version brought code and schema back in sync.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-irreversible-migration-fails-midway-02", "incident": "irreversible-migration-fails-midway", "grade": "primary", "class": "own-postmortem", "organisation": "edX", "publishedOn": "2023-02-27", "url": "https://openedx.atlassian.net/wiki/spaces/AC/pages/3686400047/RCA%3A+Enterprise+partially-applied+licensed+course+enrollment+migration", "title": "RCA: Enterprise partially-applied licensed course enrollment migration", "summary": "An edX enterprise Django migration that added tables, columns and a unique UUID field failed partway on MySQL 5.7 production and was left partially applied, blocking the deployment pipeline, and stage migrations could not be rolled back from the deploy tool.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": "load", "verifiedOn": "2026-10-21" }, { "id": "ev-irreversible-migration-fails-midway-03", "incident": "irreversible-migration-fails-midway", "grade": "primary", "class": "own-postmortem", "organisation": "FireHydrant", "publishedOn": "2021-02-23", "url": "https://firehydrant.com/blog/20210204-incident-retrospective/", "title": "February 4th, 2021 Incident Retrospective", "summary": "A FireHydrant index migration that ran outside a transaction was interrupted by an engineer after a misleading timeout error, and because it could not be rolled back the existing index was left dropped without its replacement, degrading runbook execution and application availability.", "impactClass": "degraded", "timeToDetectMinutes": 190, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-irreversible-migration-fails-midway-04", "incident": "irreversible-migration-fails-midway", "grade": "primary", "class": "own-postmortem", "organisation": "CEI", "publishedOn": "2017-06-13", "url": "https://thinkcei.atlassian.net/wiki/spaces/CEIKB/pages/76156718/June+10+2017", "title": "June 10, 2017", "summary": "A CEI ProWrite Cloud release ran a database script that completed the schema changes but failed to move data into the new tables, leaving metadata inaccessible to some users until the release was rolled back and all customer databases were restored.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": "migration", "verifiedOn": "2026-10-21" }, { "id": "ev-localisation-strings-missing-01", "incident": "localisation-strings-missing", "grade": "primary", "class": "repo-issue", "organisation": "Element", "publishedOn": "2026-02-27", "url": "https://github.com/element-hq/element-web/issues/32662", "title": "Russian translations missing in Element Web v1.12.11 interface", "summary": "Element Web v1.12.11 displayed room list strings such as Unreads and Sort in English for Russian users because the Russian translations were provided after the release candidate was cut, so they missed the release.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-localisation-strings-missing-03", "incident": "localisation-strings-missing", "grade": "primary", "class": "repo-issue", "organisation": "Thunderbird", "publishedOn": "2022-07-01", "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1777670", "title": "1777670 - TB 102 LTS - GUI not fully translated", "summary": "The Thunderbird 102 release showed many interface strings in English in the Portuguese (pt-PT) build because Fluent files added for the release were missing from the pt-PT localisation, so whole files fell back to en-US until empty files were added and the l10n changesets were updated for 102.0.2.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-long-running-migration-locks-table-01", "incident": "long-running-migration-locks-table", "grade": "primary", "class": "own-postmortem", "organisation": "Pulumi", "publishedOn": "2023-10-18", "url": "https://www.pulumi.com/blog/post-mortem-2023-10-06/", "title": "Details of the Pulumi Outage on October 6, 2023", "summary": "Pulumi shipped a database migration modifying foreign keys on a table; the resulting table copy held a lock long enough to pile up queries and exhaust database connections, and Pulumi Cloud failed 74.7% of requests for about 24 minutes.", "impactClass": "availability", "timeToDetectMinutes": 3, "testClassNamed": "migration", "verifiedOn": "2026-10-21" }, { "id": "ev-long-running-migration-locks-table-02", "incident": "long-running-migration-locks-table", "grade": "primary", "class": "status-page", "organisation": "Honeycomb", "publishedOn": "2024-08-07", "url": "https://status.honeycomb.io/incidents/ycgvhvs201q1", "title": "Honeycomb Status - UI and API unavailable", "summary": "A column addition to the teams table, applied during a deployment while a slow SELECT was running, took a metadata lock that queued all other queries on the table, including authentication, for about 20 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-long-running-migration-locks-table-03", "incident": "long-running-migration-locks-table", "grade": "primary", "class": "status-page", "organisation": "Atlassian Statuspage", "publishedOn": "2019-09-06", "url": "https://metastatuspage.com/incidents/c666y50f1xr9", "title": "Atlassian Statuspage Status - Site unavailable due to errant database migration", "summary": "An ad-hoc migration dropping legacy columns, part of an authentication change, took a table level lock that was queued behind worker job locks and blocked subsequent calls to those tables, causing a 7 minute outage of Statuspage.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-memory-leak-in-new-release-01", "incident": "memory-leak-in-new-release", "grade": "primary", "class": "status-page", "organisation": "GitHub", "publishedOn": "2023-11-03", "url": "https://www.githubstatus.com/incidents/xb30mby9fs5x", "title": "Incident with Git Operations, Issues, Pull Requests, Actions, API Requests, Codespaces, Packages, Pages and Webhooks", "summary": "A performance and resilience optimisation to GitHub's authorization microservice contained a memory leak exposed under high traffic; pods crashed repeatedly, authorization checks failed and pages returned 404s until the change was rolled back.", "impactClass": "availability", "timeToDetectMinutes": 7, "testClassNamed": "canary", "verifiedOn": "2026-10-21" }, { "id": "ev-memory-leak-in-new-release-02", "incident": "memory-leak-in-new-release", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2023-08-16", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/16187", "title": "2023-08-16: Connection errors on gitaly causing 500s (#16187)", "summary": "A new adaptive limiter deployed in Gitaly caused memory to build up on a GitLab file server from 14 August 2023 until specific Gitaly nodes became unresponsive and 3.35 percent of Git requests returned 500 errors; the commit was reverted.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-memory-leak-in-new-release-03", "incident": "memory-leak-in-new-release", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2021-09-21", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5566", "title": "2021-09-21 Gitaly goroutine memory leak on file-cny-01 since last deploy (#5566)", "summary": "After a deploy to the GitLab canary file server, a Gitaly goroutine leak caused memory to grow; a Gitaly merge request was identified as the likely cause and the server was rolled back to the previous package version.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-missing-index-after-migration-01", "incident": "missing-index-after-migration", "grade": "primary", "class": "status-page", "organisation": "Buildkite", "publishedOn": "2025-11-18", "url": "https://www.buildkitestatus.com/incidents/sv6phcn6xwwg", "title": "Increased error rate affecting Web UI and APIs", "summary": "Buildkite applied a database migration that removed an index on the annotations table; some high traffic queries did not use the replacement index, queries timed out and database CPU load rose until the index was recreated on all shards.", "impactClass": "availability", "timeToDetectMinutes": 5, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-missing-index-after-migration-02", "incident": "missing-index-after-migration", "grade": "primary", "class": "status-page", "organisation": "Onfido", "publishedOn": "2025-02-14", "url": "https://status.onfido.com/incidents/z0q8y0pvnmdk", "title": "Workflow errors and latency", "summary": "During a progressive schema migration that changed a primary key, Onfido inadvertently dropped an index, endpoint latency rose and Workflow API requests failed with database query timeouts until the index was re-created.", "impactClass": "availability", "timeToDetectMinutes": 11, "testClassNamed": "load", "verifiedOn": "2026-10-21" }, { "id": "ev-missing-index-after-migration-03", "incident": "missing-index-after-migration", "grade": "primary", "class": "status-page", "organisation": "Vapi", "publishedOn": "2024-10-02", "url": "https://status.vapi.ai/incident/438296", "title": "API is degraded", "summary": "Vapi dropped a simple index on its largest table after a concurrent compound index build had silently failed and left that index invalid, so the table had no usable index, database CPU reached 100% and API requests timed out until the index was rebuilt.", "impactClass": "availability", "timeToDetectMinutes": 5, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-missing-index-after-migration-04", "incident": "missing-index-after-migration", "grade": "primary", "class": "status-page", "organisation": "Hello Retail", "publishedOn": "2021-05-06", "url": "https://status.helloretail.com/incidents/k51j1259f86h", "title": "Stability issues", "summary": "A fault in Hello Retail's migration system removed an index related to all products during a release, and recreating the index overloaded a database cluster, which took several services down and left degraded response times for the rest of the day.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-missing-index-after-migration-05", "incident": "missing-index-after-migration", "grade": "primary", "class": "own-postmortem", "organisation": "FireHydrant", "publishedOn": "2021-02-23", "url": "https://firehydrant.com/blog/20210204-incident-retrospective/", "title": "February 4th, 2021 Incident Retrospective", "summary": "A FireHydrant migration to replace an index with a compound index dropped the existing index before the new one was built, and the missing index slowed CloudTrail ingestion jobs that then consumed background job capacity, delaying runbook steps and causing request timeouts.", "impactClass": "degraded", "timeToDetectMinutes": 190, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-oauth-callback-url-not-whitelisted-01", "incident": "oauth-callback-url-not-whitelisted", "grade": "primary", "class": "status-page", "organisation": "Phrase", "publishedOn": "2024-02-06", "url": "https://status.phrase.com/incidents/18k7jqnh3xnk", "title": "Unable to log in via Strings SSO login (redirect issue)", "summary": "A change to the Strings SAML SSO init process moved it from the legacy sso.phrase.com host to app.phrase.com and dropped the customer id from the callback path, so the redirect_uri no longer matched what customers' identity providers expected and customers could not log in via SAML for about three days.", "impactClass": "availability", "timeToDetectMinutes": 4285, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-oauth-callback-url-not-whitelisted-03", "incident": "oauth-callback-url-not-whitelisted", "grade": "practitioner", "class": "engineering-blog", "organisation": "Work 365", "publishedOn": "2025-11-03", "url": "https://support.work365apps.com/support/solutions/articles/150000215375-self-service-portal-sign-in-error-aadsts50011", "title": "Self-Service Portal Sign-In Error: AADSTS50011", "summary": "A support article describes the AADSTS50011 redirect URI mismatch seen when a Power Pages portal moves to a custom domain or is fronted by a CDN or gateway without the new host being added as a Redirect URI in the Entra ID app registration and portal Site Settings.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-oauth-callback-url-not-whitelisted-04", "incident": "oauth-callback-url-not-whitelisted", "grade": "primary", "class": "status-page", "organisation": "OutSystems", "publishedOn": "2026-01-16", "url": "https://status.outsystems.com/incidents/yc6nbbjlnxdn", "title": "Incident affecting login via custom domains", "summary": "On 16 January 2026 a service change related to an ongoing migration left OutSystems customers unable to log in to applications on newly configured custom domains, which returned an invalid redirect URI error, until custom domain redirection was temporarily disabled and affected domains were recreated.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-oauth-callback-url-not-whitelisted-05", "incident": "oauth-callback-url-not-whitelisted", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2016-06-07", "url": "https://gitlab.com/gitlab-org/gitlab/-/issues/650", "title": "Geo: The redirect URI included is not valid. (OAuth)", "summary": "A refactor of GitLab's Single Sign Out feature made GitLab Geo use a different OAuth callback URL from the one its routes used, so OAuth sign-in for Geo failed with the error that the redirect URI included is not valid.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-orm-upgrade-changes-query-semantics-01", "incident": "orm-upgrade-changes-query-semantics", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2013-03-19", "url": "https://github.blog/news-insights/the-library/today-s-email-incident/", "title": "Today's Email Incident", "summary": "After GitHub applied a Rails security release, an ActiveRecord scope began overriding the condition of an association, so a renewal email query returned teams from other organisations and the email went to many more GitHub Enterprise customer addresses than intended.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-orm-upgrade-changes-query-semantics-02", "incident": "orm-upgrade-changes-query-semantics", "grade": "primary", "class": "repo-issue", "organisation": "Mongoose", "publishedOn": "2022-09-03", "url": "https://github.com/Automattic/mongoose/issues/12389", "title": "mongoose ver 6 deleteMany() removes ALL documents when search options contains field not from schema", "summary": "A user reported that in Mongoose 6, deleteMany with a filter on a field not declared in the schema removed every document in the collection, unlike version 5; maintainers attributed it to the version 6 strictQuery default and said they would change it back in 7.0.0.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-orm-upgrade-changes-query-semantics-03", "incident": "orm-upgrade-changes-query-semantics", "grade": "practitioner", "class": "engineering-blog", "organisation": "Misty De Meo (The Future Is Now)", "publishedOn": "2025-05-13", "url": "https://www.mistys-internet.website/blog/blog/2025/05/13/the-database-row-that-did-and-didnt-exist", "title": "The database row that did and didn't exist", "summary": "After an upgrade to Django 5, saving existing records with ids beyond the int range raised IntegrityError in a QA environment, because model definitions declared int primary keys while the database used bigint and Django 5 returns an empty queryset for overflowing integer filters.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-orm-upgrade-changes-query-semantics-04", "incident": "orm-upgrade-changes-query-semantics", "grade": "primary", "class": "repo-issue", "organisation": "Django", "publishedOn": "2026-09-01", "url": "https://code.djangoproject.com/ticket/37311", "title": "#37311 (In/Range lookups on annotations consume iterator rhs)", "summary": "After the Django 6.1 release, __in lookups on annotations or aliases passed an iterator returned an empty queryset and __range lookups raised ValueError, because a change in 6.1 iterated the right-hand side twice; the fix was committed and backported for 6.1.1.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-partial-rollback-leaves-inconsistent-state-01", "incident": "partial-rollback-leaves-inconsistent-state", "grade": "primary", "class": "own-postmortem", "organisation": "Wikimedia Foundation", "publishedOn": "2021-11-08", "url": "https://wikitech.wikimedia.org/wiki/Incidents/2021-11-05_TOC_language_converter", "title": "Incidents/2021-11-05 TOC language converter", "summary": "Wikimedia rolled the MediaWiki train back to the previous version because of a table of contents regression, but the parser cache still held output written by the newer version, which the older code could not handle, so pages lost their table of contents on all wikis until the train was rolled forward again.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": "unit", "verifiedOn": "2026-10-21" }, { "id": "ev-partial-rollback-leaves-inconsistent-state-02", "incident": "partial-rollback-leaves-inconsistent-state", "grade": "primary", "class": "status-page", "organisation": "Virtuoso", "publishedOn": "2024-07-17", "url": "https://status.virtuoso.qa/incidents/2ktywjmzg968", "title": "Increased error rate on our API", "summary": "A regular platform update degraded the Virtuoso EU production environment, the automatic rollback returned only a few services to the previous version while others could not initialise the new versions, and a manual downgrade at 07:20 UTC restored full availability at 08:10 UTC.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-partial-rollback-leaves-inconsistent-state-04", "incident": "partial-rollback-leaves-inconsistent-state", "grade": "primary", "class": "regulator", "organisation": "Knight Capital Americas LLC", "publishedOn": "2013-10-16", "url": "https://www.sec.gov/files/litigation/admin/2013/34-70694.pdf", "title": "In the Matter of Knight Capital Americas LLC, Release No. 70694", "summary": "Knight deployed new RLP code to seven of eight SMARS servers, the eighth kept defective Power Peg code that the repurposed flag activated, and when Knight uninstalled the new code from the seven servers during the incident, incoming orders activated the Power Peg code on those servers too.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-password-reset-link-broken-01", "incident": "password-reset-link-broken", "grade": "primary", "class": "repo-issue", "organisation": "WP-Members", "publishedOn": "2024-10-21", "url": "https://wordpress.org/support/topic/bug-with-reset_link-link-in-reset-email/", "title": "Bug with [reset_link] link in reset email", "summary": "After the WP-Members 3.4.9.6 release, the link in the password reset email opened the reset request screen instead of the password reset screen, and the plugin author fixed the URL escaping in 3.4.9.7.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-password-reset-link-broken-02", "incident": "password-reset-link-broken", "grade": "primary", "class": "repo-issue", "organisation": "Public Knowledge Project", "publishedOn": "2023-06-26", "url": "https://github.com/pkp/pkp-lib/issues/9111", "title": "Wrong variable name in PASSWORD_RESET_CONFIRM email template", "summary": "In OJS 3.4 the password reset confirmation email template referenced a renamed variable that was not substituted, so the reset email did not carry a working reset link on fresh installs.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-password-reset-link-broken-04", "incident": "password-reset-link-broken", "grade": "primary", "class": "repo-issue", "organisation": "Qt Project", "publishedOn": "2012-01-27", "url": "https://bugreports.qt.io/rest/api/2/issue/QTJIRA-152", "title": "Reset password email contains an obsolete link leading to 404 error", "summary": "After the Qt bug tracker moved to the bugreports.qt-project.org domain, password reset emails still linked to the old domain and the link returned a 404 until the application URL was updated.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-permission-escalation-via-role-change-01", "incident": "permission-escalation-via-role-change", "grade": "primary", "class": "own-postmortem", "organisation": "Linear", "publishedOn": "2026-04-06", "url": "https://linear.app/now/linear-incident-on-mar-24th-2026", "title": "Post mortem on Linear security incident on March 24th, 2026", "summary": "A deployed change to the access control layer skipped team-level permission filtering for all but one workspace, so for about one hour members and guests could access data from private teams in their own workspace.", "impactClass": "data", "timeToDetectMinutes": 63, "testClassNamed": "integration", "verifiedOn": "2026-10-21" }, { "id": "ev-permission-escalation-via-role-change-02", "incident": "permission-escalation-via-role-change", "grade": "primary", "class": "status-page", "organisation": "Templafy", "publishedOn": "2025-10-21", "url": "https://status.templafy.com/incidents/pm47p604bz1h", "title": "Service degradation - A subset of users could encounter access issues to Spaces", "summary": "A Library App update changed the authorization logic used when fetching Space data, so users could view Spaces in their own tenant without being members until the change was rolled back across all production clusters.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-permission-escalation-via-role-change-03", "incident": "permission-escalation-via-role-change", "grade": "primary", "class": "own-postmortem", "organisation": "Workspace 365", "publishedOn": "2026-02-20", "url": "https://support.workspace365.net/en/articles/175434-post-mortem-security-misconfiguration-3-63-1", "title": "Post Mortem - Security misconfiguration - 3.63.1", "summary": "Release 3.51.0 replaced a settings page with a new User provisioning page whose endpoints lacked the admin authorization policy, so environment users without the admin role could reach them by URL until a hotfix in July 2023.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-permission-escalation-via-role-change-04", "incident": "permission-escalation-via-role-change", "grade": "primary", "class": "own-postmortem", "organisation": "JetBrains", "publishedOn": "2015-08-10", "url": "https://blog.jetbrains.com/youtrack/2015/08/jetbrains-youtrack-instance-security-issue-post-mortem/", "title": "JetBrains YouTrack Instance Security Issue Post Mortem", "summary": "An administration failure during a permission synchronization process gave all external users and guest accounts permissions to every project on the company's production YouTrack instance, including internal ones, for about an hour.", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": "security", "verifiedOn": "2026-10-21" }, { "id": "ev-provider-api-change-breaks-contract-01", "incident": "provider-api-change-breaks-contract", "grade": "primary", "class": "own-postmortem", "organisation": "CircleCI", "publishedOn": "2022-03-14", "url": "https://discuss.circleci.com/t/post-mortem-workflows-not-running-and-jobs-failing-01-03-2022/43244", "title": "Post Mortem: Workflows not running and jobs failing 01-03-2022", "summary": "GitHub moved an API endpoint that CircleCI used to check context permissions and started a 12-hour brownout of the old path, so CircleCI could not handle the 404 responses and jobs that accessed contexts failed for about 110 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-provider-api-change-breaks-contract-02", "incident": "provider-api-change-breaks-contract", "grade": "primary", "class": "own-postmortem", "organisation": "Mergify", "publishedOn": "2022-04-25", "url": "https://articles.mergify.com/handling-unexpected-third-part-change", "title": "Handling unexpected third party API changes", "summary": "GitHub stopped automatically closing a pull request when its branch was deleted, without any change to API URLs, payloads or headers, so Mergify failed to recreate speculative merge queue pull requests and some customers' pull requests were not queued until a workaround shipped.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": "integration", "verifiedOn": "2026-10-21" }, { "id": "ev-provider-api-change-breaks-contract-03", "incident": "provider-api-change-breaks-contract", "grade": "primary", "class": "status-page", "organisation": "Vapi", "publishedOn": "2025-06-19", "url": "https://status.vapi.ai/incident/605961", "title": "Breaking changes to Success Evaluation API Response", "summary": "A Vapi release that moved Success Evaluation to Gemini Structured Output changed the type of the call.analysis.successEvaluation field, so customer servers with strict type validation and users of Vapi Server SDKs failed to process post-call analysis events until a patch restored string-only output about 30 hours later.", "impactClass": "degraded", "timeToDetectMinutes": 1311, "testClassNamed": "integration", "verifiedOn": "2026-10-21" }, { "id": "ev-provider-api-change-breaks-contract-04", "incident": "provider-api-change-breaks-contract", "grade": "primary", "class": "own-postmortem", "organisation": "Logto", "publishedOn": "2024-03-25", "url": "https://blog.logto.io/postmortem-unpexected-iss-change", "title": "Postmortem: unexpected JWT `iss` change", "summary": "A Logto Cloud update made the JWT iss field reflect the custom domain used in the request instead of the default issuer, so customers who validated iss in their resource servers had auth checks fail until they changed their validation code.", "impactClass": "availability", "timeToDetectMinutes": 810, "testClassNamed": "integration", "verifiedOn": "2026-10-21" }, { "id": "ev-push-notification-token-reset-01", "incident": "push-notification-token-reset", "grade": "primary", "class": "status-page", "organisation": "Batch", "publishedOn": "2024-09-27", "url": "https://status.batch.com/incidents/thf35jz3lzkd", "title": "iOS push significant bounce rate", "summary": "A phased rollout of a new iOS push backend version made valid device tokens fail with apns2_device_token_not_for_topic errors for apps sharing one APNs p8 configuration, so many campaign notifications bounced until the rollout was reverted.", "impactClass": "availability", "timeToDetectMinutes": 1152, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-push-notification-token-reset-04", "incident": "push-notification-token-reset", "grade": "primary", "class": "status-page", "organisation": "SecureAuth", "publishedOn": "2021-11-16", "url": "https://status.secureauth.com/incidents/z2b2h9v7w2sk", "title": "Android Mobile App (Authenticate) Update Issue", "summary": "On some Android devices, updating the SecureAuth Authenticate app changed the Firebase Cloud Messaging token used to identify the device, which the service could not update dynamically, so push notifications for Push2Accept and Symbol2Accept login failed or stopped until users re-enrolled their devices.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rate-limit-misconfigured-01", "incident": "rate-limit-misconfigured", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2025-10-08", "url": "https://github.blog/news-insights/company-news/github-availability-report-september-2025/", "title": "GitHub Availability Report: September 2025", "summary": "A partial deployment of a feature flag meant to scale down rate limiting for a subset of Copilot users put the global rate limiter into an invalid state, so it limited 100 percent of requests with 403 errors for 25 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rate-limit-misconfigured-02", "incident": "rate-limit-misconfigured", "grade": "primary", "class": "own-postmortem", "organisation": "GitLab", "publishedOn": "2024-06-20", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/18174", "title": "Incident Review: Uptick in 429 errors, unexpected authentication errors", "summary": "A configuration change to HAProxy changed how application level rate limiting was handled, so traffic that had been bypassing rate limits, mainly allowlisted customers and Git over HTTP users, started receiving 429 errors for about four hours.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rate-limit-misconfigured-03", "incident": "rate-limit-misconfigured", "grade": "primary", "class": "status-page", "organisation": "HubSpot", "publishedOn": "2025-03-12", "url": "https://status.hubspot.com/incidents/6rj11ccf0kyv", "title": "HubSpot's public search API was unavailable for some users", "summary": "An unintended configuration change lowered the rate limits of HubSpot's public search API, and some customers could not use the API for about 70 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rate-limit-misconfigured-04", "incident": "rate-limit-misconfigured", "grade": "primary", "class": "status-page", "organisation": "Discord", "publishedOn": "2024-03-05", "url": "https://discordstatus.com/incidents/qg86chss85f8", "title": "Issue starting sessions, loading guilds", "summary": "Updated internal rate limiting stopped Discord sessions from starting until the rate limit was temporarily bypassed, after which guild loading was delayed while an internal service scaled up.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-release-marker-missing-in-monitoring-01", "incident": "release-marker-missing-in-monitoring", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2026-01-16", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/21101", "title": "Incident Review: INC-6665 Pipelines are failing to upload artifacts due to 502 Bad Gateway", "summary": "A deployment of a Cloudflare Worker in the GitLab.com HTTP router made large CI artifact uploads fail with 502 errors for about 10 hours, and because the router deploys outside the standard process with no deployment annotations in monitoring dashboards, responders found it hard to correlate the deployment with the incident.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-release-marker-missing-in-monitoring-02", "incident": "release-marker-missing-in-monitoring", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2026-01-06", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/21049", "title": "Incident Review: INC-6458: Sidekiq queueing SLO violation on urgent-cpu-bound shard", "summary": "A post-deployment migration on GitLab.com removed a required index on the ssh_signatures table, responders first drained the recent canary deployment without improvement before turning to the post-deployment migrations, and a follow-up was opened to add post-deployment migration annotations to Grafana.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-release-marker-missing-in-monitoring-03", "incident": "release-marker-missing-in-monitoring", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2021-09-01", "url": "https://github.blog/news-insights/company-news/github-availability-report-august-2021/", "title": "GitHub Availability Report: August 2021", "summary": "A bad service record pushed while setting up a new Actions microservice broke service-to-service calls in GitHub Actions, responders first reverted recent Actions deployments, and GitHub afterwards added broader visibility into what had changed recently across all Actions microservices.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rollback-fails-due-to-forward-only-change-01", "incident": "rollback-fails-due-to-forward-only-change", "grade": "primary", "class": "own-postmortem", "organisation": "CircleCI", "publishedOn": "2021-11-11", "url": "https://discuss.circleci.com/t/incident-report-november-8-2021-jobs-stuck-in-a-not-running-state/41890", "title": "Incident Report: November 8, 2021 - Jobs stuck in a \"not running\" state", "summary": "A deploy changed a field type in the job distribution database in a way that was not backwards compatible; after the team reverted it, the rows written between the two deploys could not be read by the old code, so distribution kept failing until a hand-built change made the distributor ignore the field.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rollback-fails-due-to-forward-only-change-02", "incident": "rollback-fails-due-to-forward-only-change", "grade": "primary", "class": "own-postmortem", "organisation": "GitLab", "publishedOn": "2020-09-22", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2731", "title": "2020-09-22: post-deployment migration failed on Production", "summary": "A post-deploy data migration that had passed on staging failed on GitLab.com production because production held records staging lacked; the daily deploy was blocked and a simple rollback was not possible because some data and schema changes had already landed, so the team fixed the migration code and rolled forward.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": "migration", "verifiedOn": "2026-10-21" }, { "id": "ev-rollback-fails-due-to-forward-only-change-03", "incident": "rollback-fails-due-to-forward-only-change", "grade": "primary", "class": "own-postmortem", "organisation": "Wikimedia Foundation", "publishedOn": "2021-11-08", "url": "https://wikitech.wikimedia.org/wiki/Incidents/2021-11-05_TOC_language_converter", "title": "Incidents/2021-11-05 TOC language converter", "summary": "A MediaWiki train release changed how the table of contents was stored in the ParserCache; when the train was rolled back to 1.38.0-wmf.6 on November 5, 2021, the older version could not handle the marker left in cached pages and tables of contents disappeared on all wikis, so the train was rolled forward again and a patch was deployed.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-rollback-fails-due-to-forward-only-change-04", "incident": "rollback-fails-due-to-forward-only-change", "grade": "primary", "class": "own-postmortem", "organisation": "Wikimedia Foundation", "publishedOn": "2023-06-19", "url": "https://wikitech.wikimedia.org/wiki/Incidents/2023-06-18_search_broken_on_wikidata_and_commons", "title": "Incidents/2023-06-18 search broken on wikidata and commons", "summary": "A scheduled Elasticsearch reindex for Wikidata and Commons de-duplicated analyzers and was promoted to production while the token_count_router query still referenced the removed text_search analyzer, so full-text search failed from June 17 11:30 to June 18 10:02 UTC; reverting was ruled out because it required a full re-index of more than 10 hours, and a one-line code fix was deployed instead.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-rolling-deploy-mixed-versions-01", "incident": "rolling-deploy-mixed-versions", "grade": "primary", "class": "status-page", "organisation": "Intercom", "publishedOn": "2026-01-05", "url": "https://www.finstatus.com/us-hosting/incidents/01KE789YVV0EDD4555S9R5ZXQY/write-up", "title": "Intercom teammate web app is down - US Hosting Status (Write-up)", "summary": "Intercom deployed a Rails framework configuration change to the session cookie signing algorithm that was not backwards compatible; during the gradual rollout requests alternated between old and new application servers, each version rejected the other's cookies, and teammates were logged out from about 14:07 to 14:23 UTC and again from 14:37 to 14:47 UTC.", "impactClass": "availability", "timeToDetectMinutes": 1, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-rolling-deploy-mixed-versions-05", "incident": "rolling-deploy-mixed-versions", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2021-03-04", "url": "https://gitlab.com/gitlab-com/gl-infra/production/-/issues/3851", "title": "2021-03-04: Pipeline status / MR widget is not loading in Web UI (#3851)", "summary": "A merge request that introduced a new pipeline state named preparing was deployed to the GitLab.com web canary fleet while Sidekiq still ran the previous code, so pipelines created through canary nodes got stuck until canary was drained.", "impactClass": "degraded", "timeToDetectMinutes": 197, "testClassNamed": null, "verifiedOn": "2026-10-22" }, { "id": "ev-schema-drift-between-services-01", "incident": "schema-drift-between-services", "grade": "primary", "class": "status-page", "organisation": "UiPath", "publishedOn": "2026-01-16", "url": "https://status.uipath.com/incidents/bl9580hc8rjr", "title": "Multiple Regions - Agents - Issue with Conversational agents in Agent Instance Management", "summary": "UiPath updated the Agent Instance Management service with a new Conversational Agents SDK while the matching Conversational Agents service update was delayed, so the SDK called routing endpoints that did not yet exist and users received 404 errors across regions.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-schema-drift-between-services-02", "incident": "schema-drift-between-services", "grade": "primary", "class": "status-page", "organisation": "Reducto", "publishedOn": "2026-01-16", "url": "https://status.reducto.ai/incidents/01KF40F54GZXA0RPA28XZ851MP/write-up", "title": "Increased Parse Failures", "summary": "Reducto deployed a change to an internal API contract with a rolling deployment, and during the window when services ran mismatched versions a subset of in-flight parse jobs failed until the change was reverted.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-schema-drift-between-services-03", "incident": "schema-drift-between-services", "grade": "primary", "class": "status-page", "organisation": "Harness", "publishedOn": "2023-11-28", "url": "https://status.harness.io/incidents/6yql7g5kg7v1", "title": "Harness Overview page showing 400 Error", "summary": "A Harness manager service release on Prod-2 included a dashboard service change whose API contract was incompatible with the service it consumed data from, and the Project Overview dashboard returned 400 errors until the deployment was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-schema-drift-between-services-04", "incident": "schema-drift-between-services", "grade": "primary", "class": "status-page", "organisation": "Stitch", "publishedOn": "2023-03-13", "url": "https://status.stitch.money/incidents/rqsn1ccn5dsb", "title": "Stitch is experiencing some intermittent issues on the API", "summary": "During a routine Stitch deployment not all internal services were deployed, so the interface versions they used to communicate fell out of sync and the API returned timeouts and 400 to 403 statuses for about 24 minutes until all services were redeployed.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sdk-upgrade-changes-behaviour-01", "incident": "sdk-upgrade-changes-behaviour", "grade": "primary", "class": "repo-issue", "organisation": "GitLab", "publishedOn": "2026-04-29", "url": "https://gitlab.com/gitlab-org/gitlab-runner/-/merge_requests/6691", "title": "Fix S3 cache broken on third-party providers by SDK checksum defaults", "summary": "GitLab Runner picked up AWS SDK Go v2 service/s3 v1.73.0, which changed checksum calculation and validation defaults to WhenSupported, so every GetObject request carried a new signed header and cache downloads from third-party S3-compatible providers failed with SignatureDoesNotMatch.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sdk-upgrade-changes-behaviour-04", "incident": "sdk-upgrade-changes-behaviour", "grade": "practitioner", "class": "engineering-blog", "organisation": "Not named in the report", "publishedOn": "2025-02-08", "url": "https://dev.to/be-hase/when-using-an-s3-compatible-object-storage-be-cautious-when-upgrading-sdk-for-java-2x-to-3ce3", "title": "When using an S3-compatible Object Storage, be cautious when upgrading SDK for Java 2.x to version 2.30.0 or later", "summary": "A developer reports that after upgrading the AWS SDK for Java 2.x S3 module to 2.30.0 or later, which enables data integrity protections by default, requests to an internal S3-compatible object storage started failing with HTTP 400 errors.", "impactClass": "unstated", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sdk-upgrade-changes-behaviour-05", "incident": "sdk-upgrade-changes-behaviour", "grade": "primary", "class": "status-page", "organisation": "Cloudflare", "publishedOn": "2025-01-17", "url": "https://www.cloudflarestatus.com/incidents/t5nrjmpxc1cj", "title": "AWS S3 SDK compatibility inconsistencies with R2", "summary": "AWS SDK releases turned on CRC32 checksums by default for multiple object operations, and because R2 did not support CRC32 checksums, R2 clients using the new SDK defaults received header errors until Cloudflare implemented a fix on 3 February 2025.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sdk-upgrade-changes-behaviour-06", "incident": "sdk-upgrade-changes-behaviour", "grade": "primary", "class": "repo-issue", "organisation": "Vates (Xen Orchestra)", "publishedOn": "2025-02-27", "url": "https://github.com/vatesfr/xen-orchestra/pull/8393", "title": "fix(fs): improve compatibility with backblaze", "summary": "After Xen Orchestra picked up AWS SDK for JavaScript 3.729 or later, the SDK sent checksum headers that Backblaze did not support, so S3 backups to Backblaze failed with an unsupported header error until a merged fix restored the previous SDK behaviour.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-soft-delete-flag-ignored-01", "incident": "soft-delete-flag-ignored", "grade": "primary", "class": "status-page", "organisation": "Storj", "publishedOn": "2025-09-19", "url": "https://status.storj.io/incidents/vj8ffc963nlj", "title": "Storj DCS Status - Email sending issue on AP1", "summary": "A newly enabled trial expiration escalation job moved more than 50,000 already deactivated user accounts back to pending deletion status and sent notification emails to their deactivated addresses, which bounced and led the email provider to disable the AP1 sending domain.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-soft-delete-flag-ignored-03", "incident": "soft-delete-flag-ignored", "grade": "primary", "class": "repo-issue", "organisation": "Laravel", "publishedOn": "2022-04-08", "url": "https://github.com/laravel/framework/issues/41885", "title": "Calling toBase() on relationships removes soft-deleted condition in Laravel 9 compared to Laravel 8 · Issue #41885 · laravel/framework · GitHub", "summary": "After the upgrade from Laravel 8 to Laravel 9 (reported on 9.7.0), calling toBase() on an Eloquent relationship dropped the SoftDeletes deleted_at IS NULL condition, so relationship queries built this way included soft-deleted rows until a maintainer fix applied the scopes in toBase().", "impactClass": "data", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sso-login-broken-01", "incident": "sso-login-broken", "grade": "primary", "class": "status-page", "organisation": "Phrase", "publishedOn": "2024-02-06", "url": "https://status.phrase.com/incidents/18k7jqnh3xnk", "title": "Unable to log in via Strings SSO login (redirect issue)", "summary": "A change to the SAML SSO init process moved a customer parameter from GET to POST and switched the host, so the callback redirect_uri no longer matched and customers could not log in via Strings SAML for about three days.", "impactClass": "availability", "timeToDetectMinutes": 4285, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sso-login-broken-02", "incident": "sso-login-broken", "grade": "primary", "class": "status-page", "organisation": "Keeper Security", "publishedOn": "2022-08-09", "url": "https://stspg.io/cn5tj7dykb4s", "title": "SSO Login Issue", "summary": "A change to the SAML request made to send a new SSO certificate caused some identity provider configurations to reject the request, blocking SSO login for some customers until the change was reverted.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sso-login-broken-03", "incident": "sso-login-broken", "grade": "primary", "class": "status-page", "organisation": "KnowBe4", "publishedOn": "2024-11-27", "url": "https://status.knowbe4.com/incidents/r33l26hjhrpf", "title": "KSAT - Login Issues", "summary": "An update added validation of the expiration date of SAML SSO certificates sent by customers' identity providers, and some certificates failed it, so those users could not log into the KSAT console for about 34 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sso-login-broken-04", "incident": "sso-login-broken", "grade": "primary", "class": "status-page", "organisation": "Mural", "publishedOn": "2023-07-18", "url": "https://status.mural.co/incidents/t0gkry57dntv", "title": "Invalid signature error when signing in with SSO", "summary": "A change to the SAML SSO login flow released to production caused some users to receive an invalid signature error when signing in with SSO until the change was rolled back the next day.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-sso-login-broken-05", "incident": "sso-login-broken", "grade": "primary", "class": "status-page", "organisation": "Smartsheet", "publishedOn": "2024-05-16", "url": "https://status.smartsheet.com/incidents/2jbjv68jm0sj", "title": "Investigating Partial Outage for Logging into Smartsheet using SSO/SAML", "summary": "An authentication change caused errors for users of specific SAML SSO configurations until it was reverted about two and a half hours after the first reports.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-subscription-renewal-skipped-02", "incident": "subscription-renewal-skipped", "grade": "primary", "class": "status-page", "organisation": "Chargebee", "publishedOn": "2024-03-01", "url": "https://status.chargebee.com/incidents/n71lqq7zvg32", "title": "Service Interruption in the GoCardless Gateway", "summary": "Chargebee reported that a recent change in its application caused collect-invoice and subscription renewal requests through the GoCardless gateway to fail with null pointer exceptions, and fixed the error handling.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-subscription-renewal-skipped-03", "incident": "subscription-renewal-skipped", "grade": "primary", "class": "status-page", "organisation": "MATCHi AB", "publishedOn": "2026-02-28", "url": "https://status.matchi.se/incidents/97g15p232385", "title": "Issues with automatic renewal of memberships", "summary": "MATCHi reported that after recent system-wide improvements its automated membership renewal job malfunctioned from 25 February 2026, so some auto-renewals were not processed, and it deployed fixes and re-ran the job to restore affected memberships.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-tax-calculation-regression-01", "incident": "tax-calculation-regression", "grade": "primary", "class": "status-page", "organisation": "Mews", "publishedOn": "2025-09-17", "url": "https://status.mews.com/incidents/xpwxfkxdhgy5", "title": "Issue with Mews Reservation with wrong VAT", "summary": "Mews reported that a recently introduced Rate Management feature caused incorrect VAT or tax rates to be applied to reservation items with price overrides, and its postmortem traced this to a frontend update that stored invalid tax codes.", "impactClass": "money", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-tax-calculation-regression-02", "incident": "tax-calculation-regression", "grade": "primary", "class": "repo-issue", "organisation": "Odoo", "publishedOn": "2024-03-25", "url": "https://github.com/odoo/odoo/pull/159154", "title": "[FIX] l10n_ee: tax report computes according to updated taxes", "summary": "An Odoo maintainer fix states that after the Estonian standard VAT rate changed from 20% to 22% at the start of 2024 and the Estonian tax report lines were updated, the report formula still computed tax at 0.2 of the taxable amount.", "impactClass": "compliance", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-third-party-rate-limit-hit-01", "incident": "third-party-rate-limit-hit", "grade": "primary", "class": "status-page", "organisation": "Harness", "publishedOn": "2026-02-20", "url": "https://status.harness.io/incidents/b688y7q8chtm", "title": "Git API rate limit errors", "summary": "A Harness release made webhook processing clear the GitX cache on every event, so entity fetches went straight to GitHub and exhausted GitHub API rate limits, blocking remote entities and pipelines for some enterprise customers until the release was rolled back.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-third-party-rate-limit-hit-02", "incident": "third-party-rate-limit-hit", "grade": "primary", "class": "own-postmortem", "organisation": "GitHub", "publishedOn": "2023-11-13", "url": "https://github.blog/news-insights/company-news/github-availability-report-october-2023/", "title": "GitHub Availability Report: October 2023", "summary": "Onboarding a new GitHub Codespaces region increased load on a global third-party dependency used by the regional authentication layer, which was throttled, so 67 percent of users could not create or start Codespaces for 2 hours and 49 minutes.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-third-party-rate-limit-hit-04", "incident": "third-party-rate-limit-hit", "grade": "practitioner", "class": "engineering-blog", "organisation": "Not named in the report", "publishedOn": "2023-11-21", "url": "https://zaidesanton.substack.com/p/how-a-3rd-party-api-can-ruin-you", "title": "How a 3rd Party API Can Ruin Your Weekend", "summary": "An engineering manager describes an unnamed company whose event-processing service started receiving HTTP 429 responses from a third-party API after breaching a project-wide limit that was not documented, which left thousands of events failed for about two weeks.", "impactClass": "degraded", "timeToDetectMinutes": null, "testClassNamed": "load", "verifiedOn": "2026-10-21" }, { "id": "ev-thundering-herd-on-restart-04", "incident": "thundering-herd-on-restart", "grade": "primary", "class": "status-page", "organisation": "Buildkite", "publishedOn": "2026-09-11", "url": "https://www.buildkitestatus.com/incidents/7nc7xd7zkxjx", "title": "Buildkite service disruption", "summary": "After a migration to more, smaller application processes raised Redis connection demand, Redis connection timeouts made clients reconnect and retry in a surge, autoscaling added pods that each opened new connection pools, and a later routine deployment added containers that caused a second period of errors.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" }, { "id": "ev-thundering-herd-on-restart-05", "incident": "thundering-herd-on-restart", "grade": "primary", "class": "status-page", "organisation": "Hex", "publishedOn": "2025-08-26", "url": "https://status.hex.tech/incidents/7zdshmk79406", "title": "Some users are experiencing issues accessing Hex projects", "summary": "A feature flag change made clients query the server more often, and a production deployment during peak traffic the next day triggered client reconnection attempts that saturated database connections until capacity was raised, the flag change was reverted and backend pods were reset.", "impactClass": "availability", "timeToDetectMinutes": null, "testClassNamed": null, "verifiedOn": "2026-10-21" } ]