{
  "title": "Control-Plane Meltdown Index",
  "version": "1.0.0",
  "generated_at": "2026-11-12T17:45:00-08:00",
  "timezone": "America/Los_Angeles",
  "incident_count": 21,
  "vendors": [
    "Amazon Web Services",
    "Cloudflare",
    "CrowdStrike",
    "Fastly",
    "Google Cloud",
    "Microsoft 365",
    "Microsoft Azure"
  ],
  "date_range": {
    "from": "2020-04-08",
    "to": "2026-11-12"
  },
  "disclaimer": "Every incident in this index is taken from the named vendor’s own post-incident report or status page. Quotes are verbatim excerpts. Dates, durations, regions, and services are included only when the vendor source states them; omitted fields were not sourced, not estimated. This dataset is curated, not exhaustive: an outage is listed only when the vendor attributes it to control-plane or API overload, retry amplification, cascading or metastable failure, configuration propagation, capacity exhaustion of a shared plane, or a closely related mechanism. Power-only, fiber-cut, and third-party BGP-hijack events are out of scope unless the vendor also describes a control-plane or configuration mechanism. Alignment with the researcher’s prior warnings is labeled researcher analysis, not proven causation.",
  "methodology": "Source URLs were fetched on 12 Nov 2026 (America/Los_Angeles) and retained only if they returned HTTP 200. Azure Front Door October 2025 lessons-learned wording is from Microsoft’s Tech Community post, cross-checked against the Azure status-history PIR pages. Microsoft 365 wording for 12 Nov 2026 is Microsoft’s service-health language as republished on the University of Illinois StatusHub mirror, because Microsoft’s public status page no longer showed the live banner at capture time. Duration labels prefer the vendor’s own recovery wording.",
  "taxonomy": [
    {
      "id": "retry_amplification",
      "label": "Retry amplification",
      "short": "Retries",
      "color": "#fb7185",
      "definition": "Failed or slow calls produce more client or peer retries, which increase load, which produce more failures — a retry storm, thundering herd, or metastable collapse."
    },
    {
      "id": "control_plane_coupling",
      "label": "Control-plane coupling",
      "short": "Control plane",
      "color": "#22d3ee",
      "definition": "A shared API, metadata, identity, DNS, or management plane sits on the path of many services. When it fails, the blast radius is far larger than the originating component."
    },
    {
      "id": "config_propagation",
      "label": "Config propagation",
      "short": "Config push",
      "color": "#f5b942",
      "definition": "A configuration, feature-file, content, or policy update is pushed — globally or past a protection layer — and a latent defect or incompatible metadata takes the data plane down with it."
    },
    {
      "id": "dependency_cascade",
      "label": "Dependency cascade",
      "short": "Cascade",
      "color": "#8b5cf6",
      "definition": "One impaired foundation (DNS, IAM, streams, a front door) causes a chain of failures in services that depend on it, often with health-check or failover side effects."
    },
    {
      "id": "capacity_exhaustion",
      "label": "Capacity exhaustion",
      "short": "Capacity",
      "color": "#34d399",
      "definition": "A thread limit, quota, health-check eviction, or overwhelmed core process removes capacity; remaining capacity is then overloaded, often accelerating the loss."
    }
  ],
  "incidents": [
    {
      "id": "gcp-iam-zanzibar-2020-04-08",
      "date": "2020-04-08",
      "start": "2020-04-08T06:52:00-07:00",
      "end": "2020-04-08T07:42:00-07:00",
      "duration_minutes": 50,
      "duration_label": "50m to majority mitigation",
      "vendor": "Google Cloud",
      "vendor_key": "google",
      "title": "IAM configuration change propagated globally past canary",
      "services": [
        "Cloud IAM",
        "Cloud SQL",
        "BigQuery streaming",
        "dependent Cloud APIs"
      ],
      "regions": [
        "Global (global IAM instance)"
      ],
      "scope": "Global",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "control_plane_coupling"
      ],
      "quote": "A latent issue with our canarying system allowed this specific type of configuration change to propagate globally in a rapid manner. As the configuration was pushed to production, the global Cloud IAM service quickly began to experience internal errors.",
      "source_url": "https://status.cloud.google.com/incidents/qp9NoUS3abKPFtxzwxRL",
      "source_title": "Google Cloud Service Health — IAM / Zanzibar incident, 8 Apr 2020",
      "notes": "Engineers were alerted at 06:52 US/Pacific; the change was identified at 07:27 and the mitigation finished propagating by 07:42 for a majority of internal services. Some services (external Cloud IAM API, high-availability Cloud SQL, BigQuery streaming) took longer. The later end is not used as a single official close time here.",
      "ongoing_as_of": null,
      "quotes": [
        "A latent issue with our canarying system allowed this specific type of configuration change to propagate globally in a rapid manner. As the configuration was pushed to production, the global Cloud IAM service quickly began to experience internal errors."
      ]
    },
    {
      "id": "cf-backbone-2020-07-17",
      "date": "2020-07-17",
      "start": null,
      "end": null,
      "duration_minutes": 27,
      "duration_label": "27m",
      "vendor": "Cloudflare",
      "vendor_key": "cloudflare",
      "title": "Backbone router configuration error overwhelmed Atlanta",
      "services": [
        "Cloudflare network / CDN (affected PoPs)"
      ],
      "regions": [
        "San Jose",
        "Dallas",
        "Seattle",
        "Los Angeles",
        "Chicago",
        "Washington, DC",
        "Richmond",
        "Newark",
        "Atlanta",
        "London",
        "Amsterdam",
        "Frankfurt",
        "Paris",
        "Stockholm",
        "Moscow",
        "St. Petersburg",
        "São Paulo",
        "Curitiba",
        "Porto Alegre"
      ],
      "scope": "Multi-region (about 50% traffic drop)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "capacity_exhaustion"
      ],
      "quote": "The outage occurred because, while working on an unrelated issue with a segment of the backbone from Newark to Chicago, our network engineering team updated the configuration on a router in Atlanta to alleviate congestion. This configuration contained an error that caused all traffic across our backbone to be sent to Atlanta. This quickly overwhelmed the Atlanta router and caused Cloudflare network locations connected to the backbone to fail.",
      "source_url": "https://blog.cloudflare.com/cloudflare-outage-on-july-17-2020/",
      "source_title": "Cloudflare outage on July 17, 2020",
      "notes": "Official duration: 27 minutes. Traffic dropped by about 50% across the network. Other locations continued to operate normally.",
      "ongoing_as_of": null,
      "quotes": [
        "The outage occurred because, while working on an unrelated issue with a segment of the backbone from Newark to Chicago, our network engineering team updated the configuration on a router in Atlanta to alleviate congestion. This configuration contained an error that caused all traffic across our backbone to be sent to Atlanta. This quickly overwhelmed the Atlanta router and caused Cloudflare network locations connected to the backbone to fail."
      ]
    },
    {
      "id": "aws-kinesis-2020-11-25",
      "date": "2020-11-25",
      "start": "2020-11-25T05:15:00-08:00",
      "end": "2020-11-25T22:23:00-08:00",
      "duration_minutes": 1028,
      "duration_label": "17h 8m",
      "vendor": "Amazon Web Services",
      "vendor_key": "aws",
      "title": "Kinesis front-end fleet exceeded OS thread limit",
      "services": [
        "Amazon Kinesis",
        "Amazon Cognito",
        "Amazon CloudWatch",
        "AWS Auto Scaling",
        "AWS Lambda"
      ],
      "regions": [
        "US-EAST-1 (Northern Virginia)"
      ],
      "scope": "Regional",
      "primary_mode": "capacity_exhaustion",
      "modes": [
        "capacity_exhaustion",
        "control_plane_coupling",
        "dependency_cascade"
      ],
      "quote": "The new capacity had caused all of the servers in the fleet to exceed the maximum number of threads allowed by an operating system configuration. As this limit was being exceeded, cache construction was failing to complete and front-end servers were ending up with useless shard-maps that left them unable to route requests to back-end clusters.",
      "source_url": "https://aws.amazon.com/message/11201/",
      "source_title": "Summary of the Amazon Kinesis Event in the Northern Virginia (US-EAST-1) Region",
      "notes": "Duration is from first alarms at 5:15 AM PST to “Kinesis fully returned to normal at 10:23 PM PST,” per the PES. Capacity addition that triggered the event began at 2:44 AM PST.",
      "ongoing_as_of": null,
      "quotes": [
        "The new capacity had caused all of the servers in the fleet to exceed the maximum number of threads allowed by an operating system configuration. As this limit was being exceeded, cache construction was failing to complete and front-end servers were ending up with useless shard-maps that left them unable to route requests to back-end clusters."
      ]
    },
    {
      "id": "gcp-oauth-quota-2020-12-14",
      "date": "2020-12-14",
      "start": null,
      "end": null,
      "duration_minutes": 47,
      "duration_label": "47m",
      "vendor": "Google Cloud",
      "vendor_key": "google",
      "title": "Automated quota system shrank User ID Service capacity",
      "services": [
        "Google User ID Service",
        "Google OAuth / customer-facing Google services"
      ],
      "regions": [
        "Global (customer-facing Google services requiring OAuth)"
      ],
      "scope": "Global",
      "primary_mode": "capacity_exhaustion",
      "modes": [
        "capacity_exhaustion",
        "config_propagation",
        "control_plane_coupling"
      ],
      "quote": "As part of an ongoing migration of the User ID Service to a new quota system, a change was made in October to register the User ID Service with the new quota system, but parts of the previous quota system were left in place which incorrectly reported the usage for the User ID Service as 0. An existing grace period on enforcing quota restrictions delayed the impact, which eventually expired, triggering automated quota systems to decrease the quota allowed for the User ID service and triggering this incident.",
      "source_url": "https://status.cloud.google.com/incidents/cFXPsFUnUELR8U2bQeGz",
      "source_title": "Google Cloud Service Health — User ID Service quota, 14 Dec 2020",
      "notes": "Official duration: 47 minutes on Monday 14 December 2020. Cloud service accounts used by GCP workloads were not impacted. Clock-time start and end were not taken as a single official pair beyond that duration.",
      "ongoing_as_of": null,
      "quotes": [
        "As part of an ongoing migration of the User ID Service to a new quota system, a change was made in October to register the User ID Service with the new quota system, but parts of the previous quota system were left in place which incorrectly reported the usage for the User ID Service as 0. An existing grace period on enforcing quota restrictions delayed the impact, which eventually expired, triggering automated quota systems to decrease the quota allowed for the User ID service and triggering this incident."
      ]
    },
    {
      "id": "fastly-cdn-2021-06-08",
      "date": "2021-06-08",
      "start": "2021-06-08T09:47:00+00:00",
      "end": "2021-06-08T12:35:00+00:00",
      "duration_minutes": 49,
      "duration_label": "49m to 95% of the network",
      "vendor": "Fastly",
      "vendor_key": "fastly",
      "title": "Valid customer configuration triggered a latent global bug",
      "services": [
        "Fastly CDN (global)"
      ],
      "regions": [
        "Global"
      ],
      "scope": "Global (85% of the network returned errors)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation"
      ],
      "quote": "We experienced a global outage due to an undiscovered software bug that surfaced on June 8 when it was triggered by a valid customer configuration change. We detected the disruption within one minute, then identified and isolated the cause, and disabled the configuration. Within 49 minutes, 95% of our network was operating as normal.",
      "quotes": [
        "We experienced a global outage due to an undiscovered software bug that surfaced on June 8 when it was triggered by a valid customer configuration change. We detected the disruption within one minute, then identified and isolated the cause, and disabled the configuration. Within 49 minutes, 95% of our network was operating as normal.",
        "Early June 8, a customer pushed a valid configuration change that included the specific circumstances that triggered the bug, which caused 85% of our network to return errors."
      ],
      "source_url": "https://www.fastly.com/blog/summary-of-june-8-outage",
      "source_title": "Summary of June 8 outage | Fastly",
      "notes": "Timeline (UTC): onset 09:47; 95% operating as normal within 49 minutes; incident mitigated 12:35. Duration_minutes records the official 49-minute recovery to 95%.",
      "ongoing_as_of": null
    },
    {
      "id": "aws-use1-2021-12-07",
      "date": "2021-12-07",
      "start": "2021-12-07T07:30:00-08:00",
      "end": "2021-12-07T14:22:00-08:00",
      "duration_minutes": 412,
      "duration_label": "6h 52m to network-device recovery",
      "vendor": "Amazon Web Services",
      "vendor_key": "aws",
      "title": "Internal-network surge, retries, and control-plane API failures",
      "services": [
        "EC2 control-plane APIs",
        "Elastic Load Balancing APIs",
        "Route 53 APIs",
        "AWS Management Console",
        "AWS STS",
        "Amazon CloudWatch",
        "API Gateway",
        "VPC Endpoints (S3, DynamoDB)"
      ],
      "regions": [
        "US-EAST-1 (Northern Virginia)"
      ],
      "scope": "Regional (control planes); existing data-plane workloads largely continued",
      "primary_mode": "retry_amplification",
      "modes": [
        "retry_amplification",
        "control_plane_coupling",
        "capacity_exhaustion"
      ],
      "quote": "This resulted in a large surge of connection activity that overwhelmed the networking devices between the internal network and the main AWS network, resulting in delays for communication between these networks. These delays increased latency and errors for services communicating between these networks, resulting in even more connection attempts and retries.",
      "quotes": [
        "This resulted in a large surge of connection activity that overwhelmed the networking devices between the internal network and the main AWS network, resulting in delays for communication between these networks. These delays increased latency and errors for services communicating between these networks, resulting in even more connection attempts and retries.",
        "Several AWS services experienced impact to the control planes that are used for creating and managing AWS resources."
      ],
      "source_url": "https://aws.amazon.com/message/12721/",
      "source_title": "Summary of the AWS Service Event in the Northern Virginia (US-EAST-1) Region",
      "notes": "Network devices fully recovered at 2:22 PM PST. Some services recovered later (STS full recovery 4:28 PM PST; new EC2 launches 2:40 PM PST). Duration uses the official 2:22 PM PST network-device recovery.",
      "ongoing_as_of": null
    },
    {
      "id": "cf-mcp-bgp-2022-06-21",
      "date": "2022-06-21",
      "start": "2022-06-21T06:27:00+00:00",
      "end": "2022-06-21T07:42:00+00:00",
      "duration_minutes": 75,
      "duration_label": "75m",
      "vendor": "Cloudflare",
      "vendor_key": "cloudflare",
      "title": "Network configuration change took 19 data centers offline",
      "services": [
        "Cloudflare network / CDN (19 Multi-Colo PoPs)"
      ],
      "regions": [
        "Amsterdam",
        "Atlanta",
        "Ashburn",
        "Chicago",
        "Frankfurt",
        "London",
        "Los Angeles",
        "Madrid",
        "Manchester",
        "Miami",
        "Milan",
        "Mumbai",
        "Newark",
        "Osaka",
        "São Paulo",
        "San Jose",
        "Singapore",
        "Sydney",
        "Tokyo"
      ],
      "scope": "19 of Cloudflare’s busiest data centers",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation"
      ],
      "quote": "This outage was caused by a change that was part of a long-running project to increase resilience in our busiest locations. A change to the network configuration in those locations caused an outage which started at 06:27 UTC. At 06:58 UTC the first data center was brought back online and by 07:42 UTC all data centers were online and working correctly.",
      "source_url": "https://blog.cloudflare.com/cloudflare-outage-on-june-21-2022/",
      "source_title": "Cloudflare outage on June 21, 2022",
      "notes": "Official window 06:27–07:42 UTC (75 minutes). These 19 locations handle a significant proportion of Cloudflare’s global traffic.",
      "ongoing_as_of": null,
      "quotes": [
        "This outage was caused by a change that was part of a long-running project to increase resilience in our busiest locations. A change to the network configuration in those locations caused an outage which started at 06:27 UTC. At 06:58 UTC the first data center was brought back online and by 07:42 UTC all data centers were online and working correctly."
      ]
    },
    {
      "id": "aws-lambda-2023-06-13",
      "date": "2023-06-13",
      "start": "2023-06-13T11:49:00-07:00",
      "end": "2023-06-13T15:37:00-07:00",
      "duration_minutes": 228,
      "duration_label": "3h 48m to full recovery",
      "vendor": "Amazon Web Services",
      "vendor_key": "aws",
      "title": "Lambda cell crossed a capacity threshold and hit a latent defect",
      "services": [
        "AWS Lambda",
        "Amazon STS",
        "AWS Management Console",
        "Amazon EKS",
        "Amazon Connect",
        "Amazon EventBridge"
      ],
      "regions": [
        "US-EAST-1 (Northern Virginia)"
      ],
      "scope": "Regional (one Lambda cell)",
      "primary_mode": "capacity_exhaustion",
      "modes": [
        "capacity_exhaustion",
        "control_plane_coupling",
        "dependency_cascade"
      ],
      "quote": "This triggered a latent software defect which caused Lambda Execution Environments to be successfully allocated for incoming requests, but never fully utilized by the Lambda Frontend.",
      "source_url": "https://aws.amazon.com/message/061323/",
      "source_title": "Summary of the AWS Service Event in the Northern Virginia (US-EAST-1) Region (13 Jun 2023)",
      "notes": "Invocations began returning to normal at 1:45 PM PDT; all affected services had fully recovered by 3:37 PM PDT. The defect was triggered when additional compute capacity crossed a capacity threshold that had not previously been reached within a single cell.",
      "ongoing_as_of": null,
      "quotes": [
        "This triggered a latent software defect which caused Lambda Execution Environments to be successfully allocated for incoming requests, but never fully utilized by the Lambda Frontend."
      ]
    },
    {
      "id": "crowdstrike-channel-file-291-2024-07-19",
      "date": "2024-07-19",
      "start": "2024-07-19T00:00:00-04:00",
      "end": null,
      "duration_minutes": null,
      "duration_label": null,
      "vendor": "CrowdStrike",
      "vendor_key": "crowdstrike",
      "title": "Sensor content-configuration update caused a widespread Windows outage",
      "services": [
        "CrowdStrike Falcon sensor (Windows)"
      ],
      "regions": [
        "Global (Windows hosts running the Falcon sensor)"
      ],
      "scope": "Global",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation"
      ],
      "quote": "On July 19, 2024, as part of regular operations, CrowdStrike released a content configuration update (via channel files) for the Windows sensor that resulted in a widespread outage.",
      "source_url": "https://www.crowdstrike.com/en-us/blog/channel-file-291-rca-available/",
      "source_title": "Channel File 291 Incident RCA is Available | CrowdStrike",
      "notes": "CrowdStrike reported that as of 29 Jul 2024 at 8:00 p.m. EDT, about 99% of Windows sensors were online compared to before the content update. A single continuous-impact duration is not recorded here because recovery varied by customer and was staggered. Additional official RCA: https://www.crowdstrike.com/wp-content/uploads/2024/08/Executive-Summary_Root-Cause-Analysis_Channel-File-291.pdf (HTTP 200).",
      "ongoing_as_of": null,
      "quotes": [
        "On July 19, 2024, as part of regular operations, CrowdStrike released a content configuration update (via channel files) for the Windows sensor that resulted in a widespread outage."
      ]
    },
    {
      "id": "cf-ratelimit-2024-06-20",
      "date": "2024-06-20",
      "start": null,
      "end": null,
      "duration_minutes": 114,
      "duration_label": "114m",
      "vendor": "Cloudflare",
      "vendor_key": "cloudflare",
      "title": "New DDoS rule exposed a latent rate-limiting bug",
      "services": [
        "Cloudflare CDN / HTTP"
      ],
      "regions": [
        "Many Cloudflare data centers worldwide"
      ],
      "scope": "Global (1.4–2.1% of HTTP requests received an error page at the 30-minute peak)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation"
      ],
      "quote": "As part of routine updates to our protection mechanisms, we created a new DDoS rule to prevent a specific type of abuse that we observed on our infrastructure. This DDoS rule worked as expected, however in a specific suspect traffic case it exposed a latent bug in our existing rate-limiting component.",
      "source_url": "https://blog.cloudflare.com/cloudflare-incident-on-june-20-2024/",
      "source_title": "Cloudflare incident on June 20, 2024",
      "notes": "Official total impact 114 minutes, from two independent events (this rule/bug and a backbone congestion event). Clock-time start was not taken from a single official ISO timestamp beyond that duration.",
      "ongoing_as_of": null,
      "quotes": [
        "As part of routine updates to our protection mechanisms, we created a new DDoS rule to prevent a specific type of abuse that we observed on our infrastructure. This DDoS rule worked as expected, however in a specific suspect traffic case it exposed a latent bug in our existing rate-limiting component."
      ]
    },
    {
      "id": "aws-kinesis-2024-07-30",
      "date": "2024-07-30",
      "start": "2024-07-30T14:45:00-07:00",
      "end": "2024-07-30T21:37:00-07:00",
      "duration_minutes": 412,
      "duration_label": "6h 52m",
      "vendor": "Amazon Web Services",
      "vendor_key": "aws",
      "title": "Kinesis cell-management misidentified healthy hosts during a deployment",
      "services": [
        "Amazon Kinesis Data Streams (internal cell)",
        "Amazon CloudWatch Logs",
        "Amazon Data Firehose",
        "Amazon S3 event framework",
        "Amazon ECS",
        "AWS Lambda",
        "Amazon Redshift",
        "AWS Glue"
      ],
      "regions": [
        "US-EAST-1 (Northern Virginia)"
      ],
      "scope": "Regional (one Kinesis cell; other cells in the Region continued to operate)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "capacity_exhaustion",
        "dependency_cascade"
      ],
      "quote": "The Kinesis cell management system determined that the healthy hosts were unhealthy and began redistributing shards that those hosts had been processing to other hosts. This created a spike in the rate of shard redistribution across the impacted Kinesis cell and eventually overloaded another component that is used to provision secure connections for communication to Kinesis data plane subsystems, which in turn impaired Kinesis traffic processing.",
      "source_url": "https://aws.amazon.com/message/073024/",
      "source_title": "Summary of the Amazon Kinesis Data Streams Service Event in Northern Virginia (US-EAST-1) Region",
      "notes": "Customer-visible window 2:45 PM–9:37 PM PDT. A routine deployment in a single AZ of a cell with a very large number of very low-throughput shards began at 9:09 AM PDT and triggered the cell-management failure.",
      "ongoing_as_of": null,
      "quotes": [
        "The Kinesis cell management system determined that the healthy hosts were unhealthy and began redistributing shards that those hosts had been processing to other hosts. This created a spike in the rate of shard redistribution across the impacted Kinesis cell and eventually overloaded another component that is used to provision secure connections for communication to Kinesis data plane subsystems, which in turn impaired Kinesis traffic processing."
      ]
    },
    {
      "id": "azure-afd-2024-07-30",
      "date": "2024-07-30",
      "start": "2024-07-30T11:45:00+00:00",
      "end": "2024-07-30T19:43:00+00:00",
      "duration_minutes": 478,
      "duration_label": "7h 58m (majority by 13:58 UTC)",
      "vendor": "Microsoft Azure",
      "vendor_key": "microsoft",
      "title": "Front Door: network control-plane failure plus latent routing configuration",
      "services": [
        "Azure Front Door",
        "Azure CDN",
        "Azure Portal"
      ],
      "regions": [
        "Multiple regions; congestion centered on a European site, with impact across regions"
      ],
      "scope": "Multi-region",
      "primary_mode": "control_plane_coupling",
      "modes": [
        "control_plane_coupling",
        "config_propagation"
      ],
      "quote": "At around 11:45 UTC, as the Network DDoS protection service was disengaging and resuming default traffic routing to the Azure Front Door service, the network routes could not be updated within one specific site in Europe. This happened because of Network control plane failures to that specific site, due to a local power outage. Consequently, traffic inside Europe continued to be forwarded to AFD through our DDoS protection services, instead of returning directly to AFD. These control plane failures were not caused or related to the initial DDoS trigger event – and in isolation, would not have caused any impact. However, an unrelated latent network configuration issue caused traffic from outside Europe to be routed to the DDoS protection system within Europe.",
      "source_url": "https://azure.status.microsoft/status/history/?trackingId=KTY1-HW8",
      "source_title": "Azure Post Incident Review — KTY1-HW8 (30 Jul 2024)",
      "notes": "Impact started 11:45 UTC; vast majority mitigated by 13:58 UTC; isolated connection failures for a subset of customers without retry logic until 19:43 UTC. A volumetric DDoS between 10:15 and 10:45 UTC was automatically mitigated and is described by Microsoft as a trigger, not the control-plane failure itself.",
      "ongoing_as_of": null,
      "quotes": [
        "At around 11:45 UTC, as the Network DDoS protection service was disengaging and resuming default traffic routing to the Azure Front Door service, the network routes could not be updated within one specific site in Europe. This happened because of Network control plane failures to that specific site, due to a local power outage. Consequently, traffic inside Europe continued to be forwarded to AFD through our DDoS protection services, instead of returning directly to AFD. These control plane failures were not caused or related to the initial DDoS trigger event – and in isolation, would not have caused any impact. However, an unrelated latent network configuration issue caused traffic from outside Europe to be routed to the DDoS protection system within Europe."
      ]
    },
    {
      "id": "azure-arm-gov-2024-09-27",
      "date": "2024-09-27",
      "start": "2024-09-27T14:47:00-04:00",
      "end": "2024-09-27T16:25:00-04:00",
      "duration_minutes": 98,
      "duration_label": "1h 38m",
      "vendor": "Microsoft Azure",
      "vendor_key": "microsoft",
      "title": "ARM control plane blocked by a global Cosmos DB misconfiguration",
      "services": [
        "Azure Resource Manager",
        "Azure portal",
        "Azure App Service",
        "Azure Kubernetes Service",
        "Azure Monitor",
        "Azure Storage",
        "and other ARM-dependent services in Azure Government"
      ],
      "regions": [
        "US Gov Arizona",
        "US Gov Texas",
        "US Gov Virginia"
      ],
      "scope": "Azure US Government regions",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "control_plane_coupling"
      ],
      "quote": "Upon investigating, we discovered that a backend Cosmos DB was misconfigured to block legitimate access from Azure Resource Manager (ARM). This prevented ARM from serving any requests, as the account is global. While Cosmos DB accounts can be replicated to multiple regions, configuration settings are global. When this account was misconfigured, the change immediately applied to all replicas impacting ARM in all regions.",
      "source_url": "https://azure.status.microsoft/status/history/?trackingId=HSKF-FB0",
      "source_title": "Azure Post Incident Review — HSKF-FB0 (27 Sep 2024)",
      "notes": "Window 14:47–16:25 EDT. Impacted control-plane operations (create, update, delete) in Azure Government.",
      "ongoing_as_of": null,
      "quotes": [
        "Upon investigating, we discovered that a backend Cosmos DB was misconfigured to block legitimate access from Azure Resource Manager (ARM). This prevented ARM from serving any requests, as the account is global. While Cosmos DB accounts can be replicated to multiple regions, configuration settings are global. When this account was misconfigured, the change immediately applied to all replicas impacting ARM in all regions."
      ]
    },
    {
      "id": "gcp-service-control-2025-06-12",
      "date": "2025-06-12",
      "start": "2025-06-12T10:51:00-07:00",
      "end": "2025-06-12T18:18:00-07:00",
      "duration_minutes": 447,
      "duration_label": "7h 27m",
      "vendor": "Google Cloud",
      "vendor_key": "google",
      "title": "Service Control crash loop after a blank quota-policy field went global",
      "services": [
        "Google Cloud APIs",
        "Google Workspace",
        "Google Security Operations",
        "Cloud IAM / Service Control"
      ],
      "regions": [
        "Global (all listed Google Cloud regions and multi-regions on the incident page)"
      ],
      "scope": "Global",
      "primary_mode": "control_plane_coupling",
      "modes": [
        "control_plane_coupling",
        "config_propagation",
        "capacity_exhaustion"
      ],
      "quote": "On June 12, 2025 at ~10:45am PDT, a policy change was inserted into the regional Spanner tables that Service Control uses for policies. Given the global nature of quota management, this metadata was replicated globally within seconds. This policy data contained unintended blank fields. Service Control, then regionally exercised quota checks on policies in each regional datastore. This pulled in blank fields for this respective policy change and exercised the code path that hit the null pointer causing the binaries to go into a crash loop.",
      "quotes": [
        "Google and Google Cloud APIs are served through our Google API management and control planes. Distributed regionally, these management and control planes are responsible for ensuring each API request that comes in is authorized, has the policy and appropriate checks (like quota) to meet their endpoints.",
        "On June 12, 2025 at ~10:45am PDT, a policy change was inserted into the regional Spanner tables that Service Control uses for policies. Given the global nature of quota management, this metadata was replicated globally within seconds. This policy data contained unintended blank fields. Service Control, then regionally exercised quota checks on policies in each regional datastore. This pulled in blank fields for this respective policy change and exercised the code path that hit the null pointer causing the binaries to go into a crash loop."
      ],
      "source_url": "https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW",
      "source_title": "Google Cloud Service Health — Service Control, 12 Jun 2025",
      "notes": "Official incident window: began 2025-06-12 10:51 and ended 2025-06-12 18:18 US/Pacific (7h 27m). The policy insertion is described at approximately 10:45am PDT.",
      "ongoing_as_of": null
    },
    {
      "id": "azure-afd-2025-10-09",
      "date": "2025-10-09",
      "start": "2025-10-09T07:50:00+00:00",
      "end": "2025-10-09T16:00:00+00:00",
      "duration_minutes": 490,
      "duration_label": "8h 10m (availability restored 12:50 UTC)",
      "vendor": "Microsoft Azure",
      "vendor_key": "microsoft",
      "title": "Front Door: manual metadata cleanup bypassed protection",
      "services": [
        "Azure Front Door",
        "Azure CDN",
        "Azure Portal",
        "Microsoft management portals"
      ],
      "regions": [
        "Africa (peak ~17% failure)",
        "Europe (~6%)",
        "Asia Pacific and the Middle East (~2.7%)"
      ],
      "scope": "Multi-region (Africa, Europe, APAC, Middle East)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "control_plane_coupling"
      ],
      "quote": "A manual cleanup of stuck tenant metadata bypassed our configuration protection layer, allowing incompatible metadata to propagate beyond our canary edge sites.",
      "quotes": [
        "A manual cleanup of stuck tenant metadata bypassed our configuration protection layer, allowing incompatible metadata to propagate beyond our canary edge sites. This metadata was created on October 7th, from a control-plane defect triggered by a customer configuration change.",
        "One of the trigger conditions for this incident was a software defect in the latest version of the AFD control plane which had been rolled out six weeks prior to the incident, in line with our safe deployment practices."
      ],
      "source_url": "https://azure.status.microsoft/status/history/?trackingId=QNBQ-5W8",
      "source_title": "Azure Post Incident Review — QNBQ-5W8 (9 Oct 2025)",
      "notes": "Official window 07:50–16:00 UTC; availability restored by 12:50 UTC, latency baseline by 16:00 UTC. The ‘manual cleanup’ wording is from Microsoft’s 4 Nov 2025 Tech Community post (HTTP 200): https://techcommunity.microsoft.com/blog/azurenetworkingblog/azure-front-door-implementing-lessons-learned-following-october-outages/4479416 — also https://aka.ms/AIR/QNBQ-5W8.",
      "ongoing_as_of": null
    },
    {
      "id": "aws-dynamodb-2025-10-19",
      "date": "2025-10-19",
      "start": "2025-10-19T23:48:00-07:00",
      "end": "2025-10-20T14:20:00-07:00",
      "duration_minutes": 872,
      "duration_label": "14h 32m overall (three distinct impact periods)",
      "vendor": "Amazon Web Services",
      "vendor_key": "aws",
      "title": "DynamoDB DNS automation wrote an empty record it could not repair",
      "services": [
        "Amazon DynamoDB",
        "EC2 instance launches",
        "Network Load Balancer",
        "AWS services depending on DynamoDB"
      ],
      "regions": [
        "US-EAST-1 (Northern Virginia)"
      ],
      "scope": "Regional, with cascading service impact",
      "primary_mode": "control_plane_coupling",
      "modes": [
        "control_plane_coupling",
        "dependency_cascade"
      ],
      "quote": "The root cause of this issue was a latent race condition in the DynamoDB DNS management system that resulted in an incorrect empty DNS record for the service’s regional endpoint (dynamodb.us-east-1.amazonaws.com) that the automation failed to repair.",
      "source_url": "https://aws.amazon.com/message/101925/",
      "source_title": "Summary of the Amazon DynamoDB Service Disruption in the Northern Virginia (US-EAST-1) Region",
      "notes": "Official overall window 11:48 PM PDT 19 Oct–2:20 PM PDT 20 Oct, in three periods: DynamoDB API errors 11:48 PM–2:40 AM; NLB connection errors 5:30 AM–2:09 PM (health-check failures); new EC2 launches 2:25 AM–10:36 AM.",
      "ongoing_as_of": null,
      "quotes": [
        "The root cause of this issue was a latent race condition in the DynamoDB DNS management system that resulted in an incorrect empty DNS record for the service’s regional endpoint (dynamodb.us-east-1.amazonaws.com) that the automation failed to repair."
      ]
    },
    {
      "id": "azure-afd-2025-10-29",
      "date": "2025-10-29",
      "start": "2025-10-29T15:41:00+00:00",
      "end": "2025-10-30T00:05:00+00:00",
      "duration_minutes": 504,
      "duration_label": "8h 24m",
      "vendor": "Microsoft Azure",
      "vendor_key": "microsoft",
      "title": "Front Door: incompatible metadata across two control-plane versions",
      "services": [
        "Azure Front Door",
        "Azure CDN",
        "Azure Portal",
        "Microsoft Entra",
        "Microsoft 365 (fronted services)",
        "DNS resolution for AFD-hosted apps"
      ],
      "regions": [
        "Global"
      ],
      "scope": "Global",
      "primary_mode": "control_plane_coupling",
      "modes": [
        "control_plane_coupling",
        "config_propagation",
        "dependency_cascade"
      ],
      "quote": "A specific sequence of customer configuration changes, performed across two different control plane build versions, resulted in incompatible customer configuration metadata being generated. These customer configuration changes themselves were valid and non-malicious – however they produced metadata that, when deployed to edge site servers, exposed a latent bug in the data plane.",
      "source_url": "https://azure.status.microsoft/status/history/?trackingId=YKYN-BWZ",
      "source_title": "Azure Post Incident Review — YKYN-BWZ (29–30 Oct 2025)",
      "notes": "Official window 15:41 UTC 29 Oct–00:05 UTC 30 Oct. Microsoft also published a lessons-learned post: https://techcommunity.microsoft.com/blog/azurenetworkingblog/azure-front-door-implementing-lessons-learned-following-october-outages/4479416",
      "ongoing_as_of": null,
      "quotes": [
        "A specific sequence of customer configuration changes, performed across two different control plane build versions, resulted in incompatible customer configuration metadata being generated. These customer configuration changes themselves were valid and non-malicious – however they produced metadata that, when deployed to edge site servers, exposed a latent bug in the data plane."
      ]
    },
    {
      "id": "cf-bot-feature-file-2025-11-18",
      "date": "2025-11-18",
      "start": "2025-11-18T11:20:00+00:00",
      "end": "2025-11-18T17:06:00+00:00",
      "duration_minutes": 190,
      "duration_label": "3h 10m to core traffic; 5h 46m to all systems",
      "vendor": "Cloudflare",
      "vendor_key": "cloudflare",
      "title": "Bot Management feature file doubled and crashed traffic-routing software",
      "services": [
        "Cloudflare core proxy / CDN",
        "Workers KV",
        "Cloudflare Access"
      ],
      "regions": [
        "Global"
      ],
      "scope": "Global",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "capacity_exhaustion",
        "dependency_cascade"
      ],
      "quote": "It was triggered by a change to one of our database systems’ permissions which caused the database to output multiple entries into a “feature file” used by our Bot Management system. That feature file, in turn, doubled in size. The larger-than-expected feature file was then propagated to all the machines that make up our network. The software running on these machines to route traffic across our network reads this feature file to keep our Bot Management system up to date with ever changing threats. The software had a limit on the size of the feature file that was below its doubled size. That caused the software to fail.",
      "source_url": "https://blog.cloudflare.com/18-november-2025-outage/",
      "source_title": "Cloudflare outage on November 18, 2025",
      "notes": "Onset 11:20 UTC; core traffic largely normal by 14:30 UTC (3h 10m); all systems functioning as normal as of 17:06 UTC (5h 46m). duration_minutes records the official core-traffic recovery. Cloudflare stated the issue was not a cyber attack.",
      "ongoing_as_of": null,
      "quotes": [
        "It was triggered by a change to one of our database systems’ permissions which caused the database to output multiple entries into a “feature file” used by our Bot Management system. That feature file, in turn, doubled in size. The larger-than-expected feature file was then propagated to all the machines that make up our network. The software running on these machines to route traffic across our network reads this feature file to keep our Bot Management system up to date with ever changing threats. The software had a limit on the size of the feature file that was below its doubled size. That caused the software to fail."
      ]
    },
    {
      "id": "cf-waf-lua-2025-12-05",
      "date": "2025-12-05",
      "start": "2025-12-05T08:47:00+00:00",
      "end": "2025-12-05T09:12:00+00:00",
      "duration_minutes": 25,
      "duration_label": "25m",
      "vendor": "Cloudflare",
      "vendor_key": "cloudflare",
      "title": "WAF change for a React vulnerability hit a latent Lua bug",
      "services": [
        "Cloudflare HTTP / WAF (older FL1 proxy)"
      ],
      "regions": [
        "Global (a subset of customers; about 28% of HTTP traffic)"
      ],
      "scope": "Partial global (~28% of HTTP traffic)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation"
      ],
      "quote": "The issue was not caused, directly or indirectly, by a cyber attack on Cloudflare’s systems or malicious activity of any kind. Instead, it was triggered by changes being made to our body parsing logic while attempting to detect and mitigate an industry-wide vulnerability disclosed this week in React Server Components.",
      "source_url": "https://blog.cloudflare.com/5-december-2025-outage/",
      "source_title": "Cloudflare outage on December 5, 2025",
      "notes": "Official window 08:47–09:12 UTC (~25 minutes). About 28% of HTTP traffic served by Cloudflare was in the affected subset.",
      "ongoing_as_of": null,
      "quotes": [
        "The issue was not caused, directly or indirectly, by a cyber attack on Cloudflare’s systems or malicious activity of any kind. Instead, it was triggered by changes being made to our body parsing logic while attempting to detect and mitigate an industry-wide vulnerability disclosed this week in React Server Components."
      ]
    },
    {
      "id": "gcp-gcve-2026-07-14",
      "date": "2026-07-14",
      "start": "2026-07-14T10:00:00-07:00",
      "end": "2026-07-14T20:40:00-07:00",
      "duration_minutes": 640,
      "duration_label": "10h 40m",
      "vendor": "Google Cloud",
      "vendor_key": "google",
      "title": "Network-control-plane configuration update dropped GCVE private traffic",
      "services": [
        "Google Cloud VMware Engine (Stretched Cluster)"
      ],
      "regions": [
        "australia-southeast2",
        "europe-west3"
      ],
      "scope": "Two regions (stretched-cluster zones)",
      "primary_mode": "config_propagation",
      "modes": [
        "config_propagation",
        "control_plane_coupling"
      ],
      "quote": "A network configuration update intended to prepare the cloud network infrastructure for new capabilities was deployed to the foundational network control plane. While the configuration payload itself was structurally valid, it exposed an implementation gap within the control plane's routing logic.",
      "quotes": [
        "A network configuration update intended to prepare the cloud network infrastructure for new capabilities was deployed to the foundational network control plane. While the configuration payload itself was structurally valid, it exposed an implementation gap within the control plane's routing logic.",
        "The disruption was traced to a network configuration update that introduced a conflict, causing inter-site communication failures, which triggered VMware Stretched Cluster failover events."
      ],
      "source_url": "https://status.cloud.google.com/incidents/T8gmtofFSTGT5tbhyciF",
      "source_title": "Google Cloud Service Health — GCVE Stretched Cluster, 14 Jul 2026",
      "notes": "Official: Incident Start 14 July 2026 10:00 PT, Incident End 14 July 2026 20:40 PT, Duration 10 hours 40 minutes (US/Pacific), as stated in Google’s incident report.",
      "ongoing_as_of": null
    },
    {
      "id": "ms365-2026-11-12",
      "date": "2026-11-12",
      "start": "2026-11-12T09:30:00-08:00",
      "end": null,
      "duration_minutes": null,
      "duration_label": "Ongoing as of 3:19 PM PT (no official full-restore time in the sourced notes)",
      "vendor": "Microsoft 365",
      "vendor_key": "microsoft",
      "title": "Exchange / Microsoft 365: servers unexpectedly unavailable, core process overwhelmed",
      "services": [
        "Exchange Online",
        "Outlook",
        "Microsoft Teams",
        "SharePoint Online",
        "Microsoft Copilot 365",
        "Microsoft Defender XDR",
        "Planner",
        "To Do",
        "Intune",
        "Defender for Office 365",
        "Microsoft Purview",
        "Power BI (intermittent MIP export)",
        "Microsoft 365 Apps Admin Center"
      ],
      "regions": [
        "Multi-region / users in any region (Microsoft wording)"
      ],
      "scope": "Global",
      "primary_mode": "capacity_exhaustion",
      "modes": [
        "capacity_exhaustion",
        "dependency_cascade"
      ],
      "quote": "We've determined that the elevated load isn't driven by an overall increase in traffic, but by a large number of servers being unexpectedly unavailable, which is shifting load onto the remaining infrastructure.",
      "quotes": [
        "We've determined that the elevated load isn't driven by an overall increase in traffic, but by a large number of servers being unexpectedly unavailable, which is shifting load onto the remaining infrastructure.",
        "Our investigation has found that a core process on the affected servers is becoming overwhelmed, which causes the servers to stop responding and contributes to the capacity loss."
      ],
      "source_url": "https://status.illinois.edu/incidents/982055",
      "source_title": "University of Illinois StatusHub incident 982055 (Microsoft service-health notes, 12 Nov 2026)",
      "ongoing_as_of": "2026-11-12T15:19:00-08:00",
      "notes": "Start about 9:30 AM PT (ThousandEyes public analysis cited ~17:30 UTC); the campus StatusHub incident was created at 10:24 AM PT. Official wording is Microsoft’s, republished on the StatusHub mirror (update 4 at 11:53 AM PT; update 8 at 2:21 PM PT). A StatusHub update 9 around 3:19 PM PT had not declared a full restore. Microsoft’s public status.cloud.microsoft page, captured around 4:07 PM PT the same day, no longer showed a live incident banner. No confirmed official end time is recorded."
    }
  ]
}
