{
  "page": {
    "id": "r5ljfdhtqdl1",
    "name": "Spacelift",
    "url": "https://spacelift.statuspage.io"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"GraphQL API","created_at":"2019-12-04T12:11:50.115Z","updated_at":"2026-10-10T09:38:50.292Z","position":1,"description":"Dynamic HTTP endpoints, including the GraphQL API","showcase":true,"start_date":null,"id":"2367p3nrsd54","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"Event processing","created_at":"2019-12-04T12:16:28.897Z","updated_at":"2026-10-30T11:29:32.525Z","position":2,"description":"Asynchronous event processing - notifications, job scheduling, webhooks etc.","showcase":true,"start_date":null,"id":"dcv4gs00sprz","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"Public workers","created_at":"2019-12-04T12:16:43.275Z","updated_at":"2026-10-21T01:26:09.571Z","position":3,"description":"Public worker pool status","showcase":true,"start_date":null,"id":"fgxg4hwmhlmp","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"Self Hosted","created_at":"2026-05-04T08:43:04.756Z","updated_at":"2026-05-04T08:43:25.906Z","position":4,"description":"Impact self hosted customers","showcase":false,"start_date":"2026-05-04T00:00:00.000Z","id":"9lr1r4qvzsj0","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"AWS","created_at":"2021-01-14T13:51:56.442Z","updated_at":"2026-05-04T08:43:25.954Z","position":5,"description":"Like most of the Internet, we depend on a number of AWS services, and decided to show the status of the ones that can impact the health of our service.","showcase":false,"start_date":null,"id":"z1yqwvpcsy6m","group_id":null,"page_id":"r5ljfdhtqdl1","components":["0jm18vczx1wg","1rcjrgnl66rc","921k1yrldq29","ywppwy9qbfgf","vcg36p85qv3k","p6l3q82w6jgd","frmyh8n59s04","gpl58cskh8n3","xssr5tzym5zn","wdtfwsvnvdwb","w57jfzqyf3jh"]},{"status":"operational","name":"GCP","created_at":"2021-01-14T14:17:12.602Z","updated_at":"2026-05-04T08:43:25.971Z","position":6,"description":null,"showcase":false,"start_date":null,"id":"06xld1tcg05s","group_id":null,"page_id":"r5ljfdhtqdl1","components":["dbxfjgzx2yb7"]},{"status":"operational","name":"GitHub","created_at":"2021-01-14T13:54:40.640Z","updated_at":"2026-05-04T08:43:26.000Z","position":7,"description":"Most of our customers use our automagical GitHub App integration. If GitHub is acting up, these customers may not get the best Spacelift experience, including not being able to log in at all.","showcase":false,"start_date":null,"id":"5df3d5ywwxhx","group_id":null,"page_id":"r5ljfdhtqdl1","components":["ckv44fby3sk2","fh6x20f68vp7","5gkcqmh4y819","b30h9nl2k33t","ltdvqnpg3d1j","6s75nrb1dz2t"]},{"status":"operational","name":"Stripe","created_at":"2021-01-14T15:14:01.128Z","updated_at":"2025-01-30T14:46:32.760Z","position":8,"description":"We use Stripe for payments, so Stripe outage may make it impossible for us to charge you, or for you to go premium.","showcase":false,"start_date":null,"id":"fk7k8k4wgcc9","group_id":null,"page_id":"r5ljfdhtqdl1","components":["wjd2cjl77vjy","899kcdhj3m7l"]},{"status":"operational","name":"Slack API","created_at":"2021-01-14T15:11:15.204Z","updated_at":"2026-11-11T19:03:56.524Z","position":9,"description":"For customers who integrate with Slack, a Slack outage may cause delays or issues, either with delivering Slack notifications, or processing interactions with Spacelift's Slack messages.","showcase":false,"start_date":null,"id":"8zzvrc215gd1","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"Platform UI","created_at":"2023-10-20T10:41:00.542Z","updated_at":"2025-01-30T14:46:32.788Z","position":11,"description":"Frontend application that serves the UI of platform","showcase":true,"start_date":"2023-10-20T00:00:00.000Z","id":"d7fsxmyc4943","group_id":null,"page_id":"r5ljfdhtqdl1"},{"status":"operational","name":"spacelift.io website","created_at":"2025-10-31T12:28:37.516Z","updated_at":"2026-10-10T09:38:50.409Z","position":12,"description":"The spacelift.io website contains information about the product, the ability to sign up for new accounts, and to login.","showcase":true,"start_date":"2025-10-31T00:00:00.000Z","id":"dbhdqcxrx9fp","group_id":null,"page_id":"r5ljfdhtqdl1"}],
  "incidents": [{"name":"Degradation of webhooks processing","status":"resolved","created_at":"2026-10-30T10:57:21.608Z","updated_at":"2026-10-30T11:29:32.604Z","monitoring_at":"2026-10-30T11:12:10.830Z","resolved_at":"2026-10-30T11:29:32.574Z","impact":"minor","shortlink":"https://stspg.io/zjy0n5qtkqkn","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-10-30T10:57:21.598Z","reminder_intervals":null,"id":"z2sx1xc7jvdk","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"After deploying the fix, webhooks processing has returned to normal.","created_at":"2026-10-30T11:29:32.574Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-30T11:29:32.574Z","display_at":"2026-10-30T11:29:32.574Z","deliver_notifications":true,"tweet_id":null,"id":"h7nhjd21vxny","incident_id":"z2sx1xc7jvdk","custom_tweet":null,"affected_components":[{"code":"dcv4gs00sprz","name":"Event processing","old_status":"degraded_performance","new_status":"operational"}]},{"status":"monitoring","body":"We have reverted the change that caused the problems with webhook processing and are seeing signs of recovery. We will continue to monitor it to ensure that the system has fully recovered.","created_at":"2026-10-30T11:12:10.830Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-30T11:12:10.830Z","display_at":"2026-10-30T11:12:10.830Z","deliver_notifications":true,"tweet_id":null,"id":"qkvvcwv778s5","incident_id":"z2sx1xc7jvdk","custom_tweet":null,"affected_components":[{"code":"dcv4gs00sprz","name":"Event processing","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"investigating","body":"We're currently investigating an issue that may result in some webhooks processing being delayed or failing to process correctly. We are working on reverting the change that caused the issue and will post an update as soon as possible.","created_at":"2026-10-30T10:57:21.711Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-30T10:57:21.711Z","display_at":"2026-10-30T10:57:21.711Z","deliver_notifications":true,"tweet_id":null,"id":"03s9ndm8z7n3","incident_id":"z2sx1xc7jvdk","custom_tweet":null,"affected_components":[{"code":"dcv4gs00sprz","name":"Event processing","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Event processing","created_at":"2019-12-04T12:16:28.897Z","updated_at":"2026-10-30T11:29:32.525Z","position":2,"description":"Asynchronous event processing - notifications, job scheduling, webhooks etc.","showcase":true,"start_date":null,"id":"dcv4gs00sprz","page_id":"r5ljfdhtqdl1","group_id":null}]},{"name":"404s downloading worker binary","status":"resolved","created_at":"2026-10-29T15:48:12.608Z","updated_at":"2026-10-30T09:11:40.422Z","monitoring_at":"2026-10-29T16:24:34.889Z","resolved_at":"2026-10-29T16:40:13.386Z","impact":"major","shortlink":"https://stspg.io/n06j8ndnlsmn","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-10-29T15:48:12.601Z","reminder_intervals":null,"id":"j5284nmqbvqg","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"This incident has been resolved. Runs in the EU region are starting normally again. If any of your runs failed during this period, please retrigger them","created_at":"2026-10-29T16:40:13.386Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-29T16:40:13.386Z","display_at":"2026-10-29T16:40:13.386Z","deliver_notifications":true,"tweet_id":null,"id":"qrxym7x5l7mt","incident_id":"j5284nmqbvqg","custom_tweet":null,"affected_components":null},{"status":"monitoring","body":"A fix has been applied and runs in the EU region are working again. We are monitoring the situation to make sure all runs start as expected","created_at":"2026-10-29T16:24:34.889Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-29T16:24:34.889Z","display_at":"2026-10-29T16:24:34.889Z","deliver_notifications":true,"tweet_id":null,"id":"kf483cd7yfsp","incident_id":"j5284nmqbvqg","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are investigating an issue where some runs fail to start with the error \"couldn't start worker: error populating worker binary\". Our team is actively working on this.","created_at":"2026-10-29T15:48:12.658Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-29T15:48:12.658Z","display_at":"2026-10-29T15:48:12.658Z","deliver_notifications":false,"tweet_id":null,"id":"sd8th615yh8m","incident_id":"j5284nmqbvqg","custom_tweet":null,"affected_components":null}],"postmortem_body":"**Summary**  \nBetween 15:31 UTC and 15:55 UTC on October 29th, runs for customers using ARM runners failed to start with the following error message:\n\ncouldn't start worker: couldn't start docker container: error populating worker binary: unexpected response code when querying for worker binary at [https://downloads.spacelift.io/worker-arm64/752bab1d/spacelift-worker](https://downloads.spacelift.io/worker-arm64/752bab1d/spacelift-worker): 404\n\n**Incident Timeline**\n\n* 15:31 UTC - A new deployment of the Spacelift backend was triggered.\n* 15:45 UTC - We triggered a rollback to the previous version.\n* 15:50 UTC - The fix began to take effect.\n* 15:55 UTC - The fix was fully rolled out.\n\n**Root Cause**  \nThe problem was caused by our build pipeline failing silently to upload that single file. When we triggered a new deployment of the Spacelift backend, new runs switched over to the new worker version and began to fail because the file was missing.\n\n**Mitigations**\n\n* Immediate: we rolled back to a previous working version.\n* Permanent: we have updated our deployment pipeline to ensure that it cannot silently fail to upload assets like this in future.","postmortem_body_last_updated_at":"2026-10-30T09:11:40.392Z","postmortem_ignored":false,"postmortem_published_at":"2026-10-30T09:11:40.392Z","postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"US Customers MQTT Service Disruption","status":"resolved","created_at":"2026-10-21T00:03:46.985Z","updated_at":"2026-10-28T14:10:44.350Z","monitoring_at":"2026-10-21T01:10:50.651Z","resolved_at":"2026-10-21T01:26:09.607Z","impact":"major","shortlink":"https://stspg.io/h6k0j0f0s71n","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-10-21T00:03:46.980Z","reminder_intervals":null,"id":"mmldkwks58ml","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2026-10-21T01:26:09.607Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-21T01:26:09.607Z","display_at":"2026-10-21T01:26:09.607Z","deliver_notifications":true,"tweet_id":null,"id":"6l6phg5qm8kx","incident_id":"mmldkwks58ml","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"major_outage","new_status":"operational"}]},{"status":"monitoring","body":"We deployed the fix and monitoring.","created_at":"2026-10-21T01:10:50.651Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-21T01:10:50.651Z","display_at":"2026-10-21T01:10:50.651Z","deliver_notifications":true,"tweet_id":null,"id":"y800gqbj4nvd","incident_id":"mmldkwks58ml","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"major_outage","new_status":"major_outage"}]},{"status":"identified","body":"We have the fix implemented and we will release it shortly.","created_at":"2026-10-21T00:17:33.280Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-21T00:17:33.280Z","display_at":"2026-10-21T00:17:33.280Z","deliver_notifications":true,"tweet_id":null,"id":"bc95y0xmrhlh","incident_id":"mmldkwks58ml","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"We’re seeing an increased number of errors affecting the worker polling API for customers in our US region. Our team is actively working on mitigation and will share updates as they become available.","created_at":"2026-10-21T00:03:47.073Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-21T00:03:47.073Z","display_at":"2026-10-21T00:03:47.073Z","deliver_notifications":true,"tweet_id":null,"id":"7jjg99fmv313","incident_id":"mmldkwks58ml","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"operational","new_status":"major_outage"}]}],"postmortem_body":"**Root Cause**  \nA change in a new launcher version caused it to connect by default to the EU endpoint if the region was not explicitly configured. Workers registered in other regions were then rejected at authentication. This was a human error.\n\nDetection was delayed for three reasons:\n\n* Our pre-release testing did not cover workers outside the EU region well enough.\n* Launcher authentication errors usually come from individual customer misconfiguration, so the increase was not flagged as a platform issue.\n* MQTT disconnections coincided with our planned MQTT-to-polling migration, which made the drop look expected.\n\n**Mitigations**\n\n* **Immediate:** We rolled back to a working version of the launcher\n* Before every deployment, we will now boot fresh private workers in other regions and confirm that they connect and complete a run. A failure like this should be caught before a release.\n* We are improving alerting so that authentication failures across multiple customers are flagged automatically as a potential platform issue.\n\n**Conclusion**  \nA broken binary caused launchers without a configured region to target the wrong endpoint. Gaps in regional test coverage and alerting meant we did not catch it before customers did. Additional automated checks are being put in place so that this failure would be detected before release along with improved alerting so that we are notified sooner for issues that make it past our automated checks.","postmortem_body_last_updated_at":"2026-10-28T14:10:44.326Z","postmortem_ignored":false,"postmortem_published_at":"2026-10-28T14:10:44.326Z","postmortem_notified_subscribers":true,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Public workers","created_at":"2019-12-04T12:16:43.275Z","updated_at":"2026-10-21T01:26:09.571Z","position":3,"description":"Public worker pool status","showcase":true,"start_date":null,"id":"fgxg4hwmhlmp","page_id":"r5ljfdhtqdl1","group_id":null}]},{"name":"Polling Workers Degragation","status":"resolved","created_at":"2026-10-20T14:05:07.908Z","updated_at":"2026-10-20T15:36:58.100Z","monitoring_at":null,"resolved_at":"2026-10-20T15:36:58.072Z","impact":"major","shortlink":"https://stspg.io/x3kv011kykmx","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-10-20T14:05:07.899Z","reminder_intervals":null,"id":"6v1pywvw05j5","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"Root cause: A software defect caused the worker communication service in the EU region to crash and restart repeatedly.\n\nImpacted customers: Customers whose workers use the poll-based communication mode. Runs on those workers were stuck when the run was triggered manually by a user and the stack had a server-side initialization policy attached. Other polling workers may have been briefly delayed while the service restarted.","created_at":"2026-10-20T15:36:58.072Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-20T15:36:58.072Z","display_at":"2026-10-20T15:36:58.072Z","deliver_notifications":true,"tweet_id":null,"id":"fr5x6442mwqc","incident_id":"6v1pywvw05j5","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"degraded_performance","new_status":"operational"}]},{"status":"identified","body":"We have the fix implemented and we will release it shortly.","created_at":"2026-10-20T14:49:41.077Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-20T14:49:41.077Z","display_at":"2026-10-20T14:49:41.077Z","deliver_notifications":true,"tweet_id":null,"id":"sgc29f2xkjys","incident_id":"6v1pywvw05j5","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"degraded_performance","new_status":"degraded_performance"}]},{"status":"identified","body":"We have noticed increased number of errors related to polling workers API, we are mitigating the problem.","created_at":"2026-10-20T14:05:08.040Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-20T14:05:08.040Z","display_at":"2026-10-20T14:05:08.040Z","deliver_notifications":false,"tweet_id":null,"id":"z92lrgpqqxlf","incident_id":"6v1pywvw05j5","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"operational","new_status":"degraded_performance"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Public workers","created_at":"2019-12-04T12:16:43.275Z","updated_at":"2026-10-21T01:26:09.571Z","position":3,"description":"Public worker pool status","showcase":true,"start_date":null,"id":"fgxg4hwmhlmp","page_id":"r5ljfdhtqdl1","group_id":null}]},{"name":"Spacelift SaaS EU region (app.spacelift.io) maintenance","status":"completed","created_at":"2026-09-23T09:33:30.134Z","updated_at":"2026-10-10T09:38:50.510Z","monitoring_at":null,"resolved_at":"2026-10-10T09:38:50.467Z","impact":"maintenance","shortlink":"https://stspg.io/yqtc29bz6fw6","scheduled_for":"2026-10-10T09:00:00.000Z","scheduled_until":"2026-10-10T11:00:00.000Z","scheduled_remind_prior":true,"scheduled_reminded_at":"2026-10-10T08:00:07.630Z","impact_override":null,"scheduled_auto_in_progress":true,"scheduled_auto_completed":true,"metadata":{},"started_at":"2026-09-23T09:33:30.122Z","reminder_intervals":null,"id":"q7ybj3nrvqbn","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"completed","body":"The scheduled PostgreSQL database upgrade for the EU region (app.spacelift.io) has been completed successfully. All services are fully operational, and normal platform usage may resume.\n\nIf you notice anything unexpected, please contact our support team. Thank you for your patience.","created_at":"2026-10-10T09:38:50.467Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-10T09:38:50.467Z","display_at":"2026-10-10T09:38:50.467Z","deliver_notifications":true,"tweet_id":null,"id":"2jq5mtbxg71f","incident_id":"q7ybj3nrvqbn","custom_tweet":null,"affected_components":[{"code":"2367p3nrsd54","name":"GraphQL API","old_status":"under_maintenance","new_status":"operational"},{"code":"dcv4gs00sprz","name":"Event processing","old_status":"under_maintenance","new_status":"operational"},{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"under_maintenance","new_status":"operational"},{"code":"dbhdqcxrx9fp","name":"spacelift.io website","old_status":"under_maintenance","new_status":"operational"}]},{"status":"in_progress","body":"Scheduled maintenance is currently in progress. We will provide updates as necessary.","created_at":"2026-10-10T09:00:14.496Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-10T09:00:14.496Z","display_at":"2026-10-10T09:00:14.496Z","deliver_notifications":true,"tweet_id":null,"id":"9fzw69c4xcgr","incident_id":"q7ybj3nrvqbn","custom_tweet":null,"affected_components":[{"code":"2367p3nrsd54","name":"GraphQL API","old_status":"operational","new_status":"under_maintenance"},{"code":"dcv4gs00sprz","name":"Event processing","old_status":"operational","new_status":"under_maintenance"},{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"operational","new_status":"under_maintenance"},{"code":"dbhdqcxrx9fp","name":"spacelift.io website","old_status":"operational","new_status":"under_maintenance"}]},{"status":"scheduled","body":"Maintenance will begin as scheduled in 60 minutes.","created_at":"2026-10-10T08:00:07.435Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-10-10T08:00:07.435Z","display_at":"2026-10-10T08:00:07.435Z","deliver_notifications":true,"tweet_id":null,"id":"91snpg3rfn2k","incident_id":"q7ybj3nrvqbn","custom_tweet":null,"affected_components":null},{"status":"scheduled","body":"Spacelift SaaS maintenance: Saturday, 2026-10-10, 09:00-11:00 UTC (05:00-07:00 ET, 11:00-13:00 CET). EU region only - US (us.spacelift.io) and FedRAMP (fedramp.spacelift.io) are unaffected.\n\nPurpose: PostgreSQL database upgrade.\n\nImpact: During the maintenance window, there will be an estimated 5 to 10 minutes during which no new runs will be scheduled, and a maintenance notice will appear in the app - it will be unavailable during that time.\n\nIn-progress runs: Attempt to complete automatically on a best-effort basis.\n\nQueued runs: Hold and start automatically once maintenance ends.\n\nVCS webhooks during the window: Processed automatically once maintenance ends - no need to retrigger.\n\nAction: We recommend that you do not use Spacelift during the maintenance window, for example, triggering runs or adjusting your configuration.\n\nUpdates will be posted here.","created_at":"2026-09-23T09:33:30.213Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-23T09:46:42.944Z","display_at":"2026-09-23T09:33:30.213Z","deliver_notifications":true,"tweet_id":null,"id":"b5d580xxycs0","incident_id":"q7ybj3nrvqbn","custom_tweet":null,"affected_components":[{"code":"2367p3nrsd54","name":"GraphQL API","old_status":"operational","new_status":"operational"},{"code":"dcv4gs00sprz","name":"Event processing","old_status":"operational","new_status":"operational"},{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"operational","new_status":"operational"},{"code":"dbhdqcxrx9fp","name":"spacelift.io website","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"GraphQL API","created_at":"2019-12-04T12:11:50.115Z","updated_at":"2026-10-10T09:38:50.292Z","position":1,"description":"Dynamic HTTP endpoints, including the GraphQL API","showcase":true,"start_date":null,"id":"2367p3nrsd54","page_id":"r5ljfdhtqdl1","group_id":null},{"status":"operational","name":"Event processing","created_at":"2019-12-04T12:16:28.897Z","updated_at":"2026-10-30T11:29:32.525Z","position":2,"description":"Asynchronous event processing - notifications, job scheduling, webhooks etc.","showcase":true,"start_date":null,"id":"dcv4gs00sprz","page_id":"r5ljfdhtqdl1","group_id":null},{"status":"operational","name":"Public workers","created_at":"2019-12-04T12:16:43.275Z","updated_at":"2026-10-21T01:26:09.571Z","position":3,"description":"Public worker pool status","showcase":true,"start_date":null,"id":"fgxg4hwmhlmp","page_id":"r5ljfdhtqdl1","group_id":null},{"status":"operational","name":"spacelift.io website","created_at":"2025-10-31T12:28:37.516Z","updated_at":"2026-10-10T09:38:50.409Z","position":12,"description":"The spacelift.io website contains information about the product, the ability to sign up for new accounts, and to login.","showcase":true,"start_date":"2025-10-31T00:00:00.000Z","id":"dbhdqcxrx9fp","page_id":"r5ljfdhtqdl1","group_id":null}]},{"name":"Event Processing Delays","status":"resolved","created_at":"2026-09-11T13:22:43.529Z","updated_at":"2026-09-11T13:43:11.524Z","monitoring_at":null,"resolved_at":"2026-09-11T13:43:11.512Z","impact":"major","shortlink":"https://stspg.io/nlmyx6ssfn19","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-09-11T13:22:43.521Z","reminder_intervals":null,"id":"d071z8szxn6c","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"We have restored events processing to the usual state. The root cause was a change that we released earlier today.","created_at":"2026-09-11T13:43:11.512Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-11T13:43:11.512Z","display_at":"2026-09-11T13:43:11.512Z","deliver_notifications":true,"tweet_id":null,"id":"yj64fw6cbp92","incident_id":"d071z8szxn6c","custom_tweet":null,"affected_components":[{"code":"dcv4gs00sprz","name":"Event processing","old_status":"operational","new_status":"operational"}]},{"status":"identified","body":"We're seeing increased delays and/or elevated error rates in our FedRamp event processing queue. The root cause has been identified, we working on restoring the regular functionality of the system.","created_at":"2026-09-11T13:22:43.576Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-09-11T13:22:43.576Z","display_at":"2026-09-11T13:22:43.576Z","deliver_notifications":true,"tweet_id":null,"id":"vmbkn7syy90v","incident_id":"d071z8szxn6c","custom_tweet":null,"affected_components":[{"code":"dcv4gs00sprz","name":"Event processing","old_status":"operational","new_status":"operational"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Event processing","created_at":"2019-12-04T12:16:28.897Z","updated_at":"2026-10-30T11:29:32.525Z","position":2,"description":"Asynchronous event processing - notifications, job scheduling, webhooks etc.","showcase":true,"start_date":null,"id":"dcv4gs00sprz","page_id":"r5ljfdhtqdl1","group_id":null}]},{"name":"AWS connectivity for runs using hashicorp/aws v6.57.0","status":"resolved","created_at":"2026-07-29T13:39:04.248Z","updated_at":"2026-07-29T15:44:52.553Z","monitoring_at":null,"resolved_at":"2026-07-29T15:44:52.537Z","impact":"none","shortlink":"https://stspg.io/jx79l6fcqgmp","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"none","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-07-29T13:39:04.242Z","reminder_intervals":null,"id":"2qgk4pcpj8df","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2026-07-29T15:44:52.537Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-29T15:44:52.537Z","display_at":"2026-07-29T15:44:52.537Z","deliver_notifications":true,"tweet_id":null,"id":"t1xdwvz0vbnr","incident_id":"2qgk4pcpj8df","custom_tweet":null,"affected_components":null},{"status":"identified","body":"We have identified a problem with v6.57.0 of the hashicorp/aws provider causing requests for certain resources to fail. A workaround for affected customers is to pin to v6.56.0 or below.\n\nMore details can be found on the HashiCorp status page: https://status.hashicorp.com/incidents/01KYPY1KTBE8KWC4E75RXN0XX8","created_at":"2026-07-29T15:22:42.544Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-29T15:22:42.544Z","display_at":"2026-07-29T15:22:42.544Z","deliver_notifications":true,"tweet_id":null,"id":"33zqj1h9zcr5","incident_id":"2qgk4pcpj8df","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are experiencing failing AWS API calls for runs using hashicorp/aws provider with version v6.57.0.","created_at":"2026-07-29T13:39:04.279Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-29T13:39:04.279Z","display_at":"2026-07-29T13:39:04.279Z","deliver_notifications":true,"tweet_id":null,"id":"3yg5tq2yfhp6","incident_id":"2qgk4pcpj8df","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"External symlink configuration issue","status":"resolved","created_at":"2026-07-13T15:19:44.796Z","updated_at":"2026-07-16T14:52:42.667Z","monitoring_at":null,"resolved_at":"2026-07-16T14:52:42.648Z","impact":"none","shortlink":"https://stspg.io/wbhfh8ml7ppl","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":null,"scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-07-13T15:19:44.788Z","reminder_intervals":null,"id":"gl1hq88qqxhc","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"The incident has been resolved.","created_at":"2026-07-16T14:52:42.648Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-16T14:52:42.648Z","display_at":"2026-07-16T14:52:42.648Z","deliver_notifications":false,"tweet_id":null,"id":"hgv3lkddrr3b","incident_id":"gl1hq88qqxhc","custom_tweet":null,"affected_components":null},{"status":"identified","body":"We have identified an issue with external symlink configurations affecting some setups. Please contact Support if you are affected.","created_at":"2026-07-13T15:19:44.845Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-07-13T15:19:44.845Z","display_at":"2026-07-13T15:19:44.845Z","deliver_notifications":false,"tweet_id":null,"id":"7n04wqvwnz3d","incident_id":"gl1hq88qqxhc","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Runs failing on workers","status":"resolved","created_at":"2026-06-01T08:51:41.620Z","updated_at":"2026-06-01T11:09:06.449Z","monitoring_at":null,"resolved_at":"2026-06-01T11:09:06.435Z","impact":"major","shortlink":"https://stspg.io/81yrrmdk5fr2","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-06-01T08:51:41.607Z","reminder_intervals":null,"id":"g3bg08cy6cnv","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"This incident has been resolved. A [new version](https://github.com/spacelift-io/spacelift-worker-image/releases/tag/1.82.0) of the image has been released.","created_at":"2026-06-01T11:09:06.435Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-01T11:09:06.435Z","display_at":"2026-06-01T11:09:06.435Z","deliver_notifications":false,"tweet_id":null,"id":"d00dzqn7sq3h","incident_id":"g3bg08cy6cnv","custom_tweet":null,"affected_components":null},{"status":"identified","body":"As a mitigation we have deregistered all of the AMIs for version 1.80.0 and 1.81.0. We are currently working on building new images that include a fix for the problem.","created_at":"2026-06-01T10:11:48.747Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-01T10:11:48.747Z","display_at":"2026-06-01T10:11:48.747Z","deliver_notifications":true,"tweet_id":null,"id":"66qq7lpxytkm","incident_id":"g3bg08cy6cnv","custom_tweet":null,"affected_components":null},{"status":"identified","body":"The issue has been identified. This is not affecting public workers; only private workers can be affected.\n\nThe last working version of the worker's image is [v1.79.0](https://github.com/spacelift-io/spacelift-worker-image/releases/tag/1.79.0).\nPlease rollback to that version if you see failing runs after an update.","created_at":"2026-06-01T09:22:02.542Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-01T09:22:02.542Z","display_at":"2026-06-01T09:22:02.542Z","deliver_notifications":false,"tweet_id":null,"id":"j72wm9sgntv0","incident_id":"g3bg08cy6cnv","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We've spotted as issue with failing runs. We're currently investigating the issue, and will provide an update soon.","created_at":"2026-06-01T08:51:41.739Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-06-01T08:51:41.739Z","display_at":"2026-06-01T08:51:41.739Z","deliver_notifications":true,"tweet_id":null,"id":"rl4tm3sppcl3","incident_id":"g3bg08cy6cnv","custom_tweet":null,"affected_components":[{"code":"fgxg4hwmhlmp","name":"Public workers","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Terraform runs failing due to expired HashiCorp signing key","status":"resolved","created_at":"2026-04-20T11:43:52.000Z","updated_at":"2026-04-21T09:20:10.186Z","monitoring_at":"2026-04-20T11:43:52.000Z","resolved_at":"2026-04-21T09:20:10.166Z","impact":"major","shortlink":"https://stspg.io/r338b6b7dqgx","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2026-04-20T11:43:52.000Z","reminder_intervals":null,"id":"r17qd6sqn7g7","page_id":"r5ljfdhtqdl1","incident_updates":[{"status":"resolved","body":"If self hosted customer are still affected, please reach out to our support team.","created_at":"2026-04-21T09:20:10.166Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-21T09:20:10.166Z","display_at":"2026-04-21T09:20:10.166Z","deliver_notifications":true,"tweet_id":null,"id":"kws8vbqgr89n","incident_id":"r17qd6sqn7g7","custom_tweet":null,"affected_components":null},{"status":"monitoring","body":"We are continuing to monitor for any further issues.","created_at":"2026-04-21T08:32:26.522Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-21T08:32:26.522Z","display_at":"2026-04-21T08:32:26.522Z","deliver_notifications":false,"tweet_id":null,"id":"k7cc3y708h6q","incident_id":"r17qd6sqn7g7","custom_tweet":null,"affected_components":null},{"status":"monitoring","body":"We identified that Terraform runs were failing when downloading the Terraform binary. The public GPG key we use to verify HashiCorp's checksum signatures had expired, causing every binary download to fail signature verification. \nRuns using OpenTofu or a Custom workflow were not affected.\n\nA fix updating the embedded HashiCorp public key has been rolled out during the week-end across our managed infrastructure. \nIf you are using private worker pools, you will need to cycle your worker pool so the workers pick up the latest launcher version that includes the fix. Until workers are cycled, new Terraform runs on those pools will continue to fail.\nCycling worker pool is not required if using kubernetes native workers. This is only to make sure EC2 docker based workers are refreshed.\n\nAction required for Self-Hosted: Hotfix v5.1.1 has been published and can be used to resolve the issue on Self-Hosted installations on v5.1.0. If you are on an older Self-Hosted version, please contact your Spacelift account team — we are publishing targeted hotfixes on request.\n\nIf you cannot upgrade right now, you can unblock Terraform runs by:\n- Switch from Terraform to OpenTofu if you can.\n- Switch to a custom workflow for your stack and provide Terraform binary yourself\n- Setting \"skipVendorBinaryDownload: true\" on the affected stacks, and running those stacks in a runner image that already has the terraform binary installed at the expected path.","created_at":"2026-04-20T11:43:52.608Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2026-04-21T08:32:09.112Z","display_at":"2026-04-20T11:43:52.000Z","deliver_notifications":true,"tweet_id":null,"id":"xf642h70z22x","incident_id":"r17qd6sqn7g7","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]}]
}