[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"sanity-MiYIhu2C1DgYmE13pZq2YPAEtvHR2Us98Sr6bbjNyeM":3,"sanity-khiT1PyyujHmnCg_X3PDmjAyC4o5LG6uyYTJeTPQyC0":378},{"data":4,"sourceMap":-1},{"latestPodcast":5,"latestReleases":14,"post":39,"recent":353},[6],{"_id":7,"publishedAt":8,"slug":9,"sponsored":12,"title":13},"f83eb5f0-1237-487f-84d8-f7abf2318c39","2026-06-25T07:40:00.000Z",{"_type":10,"current":11},"slug","code-isnt-causing-your-production-failures",null,"Code isn’t the only thing causing your production failures",[15,21,27,33],{"_id":16,"publishedAt":17,"slug":18,"title":20},"eb5b66eb-9410-4329-83bb-22bbff39402a","2026-04-28T13:00:00.000Z",{"_type":10,"current":19},"turn-scattered-knowledge-into-trusted-intelligence","Turning scattered knowledge into trusted intelligence: Stack Internal 2026.3",{"_id":22,"publishedAt":23,"slug":24,"title":26},"369c2401-b62e-4a37-8ff8-bf603023ecad","2026-03-02T15:03:00.988Z",{"_type":10,"current":25},"what-s-new-at-stack-overflow-march-2026","What’s new at Stack Overflow: March 2026",{"_id":28,"publishedAt":29,"slug":30,"title":32},"5e9053a4-07ea-447c-91ea-29e0b6228537","2026-02-02T15:00:00.000Z",{"_type":10,"current":31},"what-s-new-at-stack-overflow-february-2026","What’s new at Stack Overflow: February 2026",{"_id":34,"publishedAt":35,"slug":36,"title":38},"a1b538eb-a8a6-46d0-80a1-ac70ec9bb935","2026-01-05T10:00:00.000-05:00",{"_type":10,"current":37},"what-s-new-at-stack-overflow-january-2026","What’s new at Stack Overflow: January 2026",{"_createdAt":40,"_id":41,"_rev":42,"_type":43,"_updatedAt":44,"author":45,"body":59,"comments":325,"dateUrl":326,"excerpt":172,"legacyBody":327,"product":12,"publishedAt":330,"slug":331,"sponsored":12,"tags":333,"title":352,"visible":325},"2023-05-25T09:36:59Z","wp-post-3634","dgl3SCUzppW3U2LvCoS3kq","blogPost","2023-07-13T14:54:30Z",[46],{"_createdAt":47,"_id":48,"_rev":49,"_type":50,"_updatedAt":51,"avatar":52,"employee":54,"name":55,"role":56,"slug":57},"2023-05-23T16:27:18Z","wp-author-114","07ZbrKPSUrjrV4wQ6fam8u","blogAuthor","2023-08-29T11:49:01Z",{"_type":53},"image","former","Jeff Atwood","Co-founder",{"current":58},"jeffatwood",[60,92,100,119,127,135,151,166,173,181,189,209,217,236,244,274,282,298,306],{"_key":61,"_type":62,"children":63,"markDefs":87,"style":91},"2b3259beb150","block",[64,69,74,78,83],{"_key":65,"_type":66,"marks":67,"text":68},"2b3259beb1500","span",[],"On Friday, the server which hosts this blog suffered catastrophic data loss. Fortunately, the blog server is at ",{"_key":70,"_type":66,"marks":71,"text":73},"2b3259beb1501",[72],"strong","a different datacenter entirely",{"_key":75,"_type":66,"marks":76,"text":77},"2b3259beb1502",[]," than ",{"_key":79,"_type":66,"marks":80,"text":82},"2b3259beb1503",[81],"ae6328521ca8","PEAK, which hosts all the Trilogy sites",{"_key":84,"_type":66,"marks":85,"text":86},"2b3259beb1504",[],".",[88],{"_key":81,"_type":89,"href":90,"reference":12},"link","http://blog.stackoverflow.com/2009/02/new-datacenter-migration/","normal",{"_key":93,"_type":62,"children":94,"markDefs":99,"style":91},"05b567dfa572",[95],{"_key":96,"_type":66,"marks":97,"text":98},"05b567dfa5720",[],"It's a long story, and I'll document it in more detail elsewhere, but the short version is this:",[],{"_key":101,"_type":62,"children":102,"level":116,"listItem":117,"markDefs":118,"style":91},"9ce6139d80f7",[103,107,112],{"_key":104,"_type":66,"marks":105,"text":106},"9ce6139d80f70",[],"This particular host's (again, ",{"_key":108,"_type":66,"marks":109,"text":111},"9ce6139d80f71",[110],"em","not",{"_key":113,"_type":66,"marks":114,"text":115},"9ce6139d80f72",[]," PEAK) backup processes were fatally flawed, as they were unable to backup a live virtual machine hard drive file. So the \"backups\" they had were nonexistent, because the backup process was failing on the most important server files every single night.",1,"bullet",[],{"_key":120,"_type":62,"children":121,"level":116,"listItem":117,"markDefs":126,"style":91},"d9593b2ef2f4",[122],{"_key":123,"_type":66,"marks":124,"text":125},"d9593b2ef2f40",[],"We belatedly realized we should never have trusted the host with backups in the first place -- we should have been backing up the relevant bits of content in the virtual machine ourselves, instead of assuming the host's backups were working.",[],{"_key":128,"_type":62,"children":129,"level":116,"listItem":117,"markDefs":134,"style":91},"3916ba9780bf",[130],{"_key":131,"_type":66,"marks":132,"text":133},"3916ba9780bf0",[],"The blog was not a high priority for backup since it was a) on a singleton server in a completely different data center and b) it's not an actual trilogy site, but a \"helper\" site.",[],{"_key":136,"_type":62,"children":137,"markDefs":150,"style":91},"e65088febb20",[138,142,146],{"_key":139,"_type":66,"marks":140,"text":141},"e65088febb200",[],"Anyway, we were able to piece together a backup from different sources, and I took this opportunity to move the blog from WordPress on Windows (which has been ",{"_key":143,"_type":66,"marks":144,"text":145},"e65088febb201",[110],"incredibly",{"_key":147,"_type":66,"marks":148,"text":149},"e65088febb202",[]," quirky) to WordPress on Linux.",[],{"_key":152,"_type":62,"children":153,"markDefs":165,"style":91},"a86fcaea3081",[154,158,162],{"_key":155,"_type":66,"marks":156,"text":157},"a86fcaea30810",[],"You may be wondering how this incident relates to our ",{"_key":159,"_type":66,"marks":160,"text":161},"a86fcaea30811",[72],"Stack Overflow disaster recovery plan",{"_key":163,"_type":66,"marks":164,"text":86},"a86fcaea30812",[],[],{"_key":167,"_type":53,"alt":168,"asset":169,"caption":172,"markDefs":12},"03499d69dc64","our-disaster-recovery-plan",{"_ref":170,"_type":171},"image-a5f53201806445bd66ab1165e7b093d557229cdf-475x421-png","reference","",{"_key":174,"_type":62,"children":175,"markDefs":180,"style":91},"ceaf70238c40",[176],{"_key":177,"_type":66,"marks":178,"text":179},"ceaf70238c400",[],"Mostly, it doesn't. I just never viewed the blog as part of our core mission (though I probably should have) and subsequently didn't give it the attention it deserved. We've since moved the blog to an actual PEAK \"family\" server in our rack, so it'll get folded in with our standard backup process.",[],{"_key":182,"_type":62,"children":183,"markDefs":188,"style":91},"ec475cdab6e5",[184],{"_key":185,"_type":66,"marks":186,"text":187},"ec475cdab6e50",[],"So what is our standard backup process?",[],{"_key":190,"_type":62,"children":191,"level":116,"listItem":205,"markDefs":206,"style":91},"117dc8ba8f8e",[192,196,201],{"_key":193,"_type":66,"marks":194,"text":195},"117dc8ba8f8e0",[],"We take full database backups of all databases at 4 AM, 4 PM, and 12 AM. (some databases are backed up more aggressively, but this is typical.) These full database backups are stored on ",{"_key":197,"_type":66,"marks":198,"text":200},"117dc8ba8f8e1",[199],"2b322cd47d63","our NAS RAID-6 device",{"_key":202,"_type":66,"marks":203,"text":204},"117dc8ba8f8e2",[]," on the rack at the PEAK datacenter.","number",[207],{"_key":199,"_type":89,"href":208,"reference":12},"http://blog.stackoverflow.com/2009/02/our-backup-strategy-inexpensive-nas/",{"_key":210,"_type":62,"children":211,"level":116,"listItem":205,"markDefs":216,"style":91},"6680c41baec9",[212],{"_key":213,"_type":66,"marks":214,"text":215},"6680c41baec90",[],"We have a 500 GB USB hard drive attached to the database server. There is a C# script which copies the latest backups from the NAS to the USB hard drive every night at around 1 AM. The oldest files are deleted to make room for the new files as necessary. (The current Stack Overflow full backup is about 7 GB compressed, and the other databases are perhaps 2 GB compressed.) new: we'll have two USB hard drives connected and do identical copies in parallel in case one of the drives develops problems.",[],{"_key":218,"_type":62,"children":219,"level":116,"listItem":205,"markDefs":233,"style":91},"731aa0280022",[220,224,229],{"_key":221,"_type":66,"marks":222,"text":223},"731aa02800220",[],"One of our team members, ",{"_key":225,"_type":66,"marks":226,"text":228},"731aa02800221",[227],"3e179db6f2ac","Geoff Dalgas",{"_key":230,"_type":66,"marks":231,"text":232},"731aa02800222",[],", lives a mile from the PEAK data center. He drops by and physically swaps out the USB hard drive every few weeks. He holds onefour 500 GB USB drives at his home, while the other two are at the data center. They continually get cycled back and forth over time.",[234],{"_key":227,"_type":89,"href":235,"reference":12},"http://blog.stackoverflow.com/2009/05/welcome-stack-overflow-valued-associate-00003/",{"_key":237,"_type":62,"children":238,"level":116,"listItem":205,"markDefs":243,"style":91},"e6bc029ebe53",[239],{"_key":240,"_type":66,"marks":241,"text":242},"e6bc029ebe530",[],"new: Fog Creek will FTP in and transfer the most current database backups to their hosting facility every week, during low traffic periods on Saturday.",[],{"_key":245,"_type":62,"children":246,"level":116,"listItem":205,"markDefs":269,"style":91},"d7ffde05b26f",[247,251,256,260,265],{"_key":248,"_type":66,"marks":249,"text":250},"d7ffde05b26f0",[],"We do ",{"_key":252,"_type":66,"marks":253,"text":255},"d7ffde05b26f1",[254],"ac166498a001","Creative Commons data dumps",{"_key":257,"_type":66,"marks":258,"text":259},"d7ffde05b26f2",[]," of all sites (Stack Overflow, Server Fault, Super User) every month. This is a subset of the data, but a sizable one, and it's available on ",{"_key":261,"_type":66,"marks":262,"text":264},"d7ffde05b26f3",[263],"05b7a439ac8a","Legal Torrents",{"_key":266,"_type":66,"marks":267,"text":268},"d7ffde05b26f4",[],". These data dumps are physically hosted on and seeded by Legal Torrents.",[270,272],{"_key":254,"_type":89,"href":271,"reference":12},"http://blog.stackoverflow.com/category/cc-wiki-dump/",{"_key":263,"_type":89,"href":273,"reference":12},"http://www.legaltorrents.com/creators/146-stack-overflow-data-dump",{"_key":275,"_type":62,"children":276,"level":116,"listItem":205,"markDefs":281,"style":91},"1f7991776511",[277],{"_key":278,"_type":66,"marks":279,"text":280},"1f79917765110",[],"Our Subversion source control repository is copied to the NAS every day and also gets copied to the USB external drive, etc, through the same script.",[],{"_key":283,"_type":62,"children":284,"level":116,"listItem":205,"markDefs":297,"style":91},"ff436c10e722",[285,289,293],{"_key":286,"_type":66,"marks":287,"text":288},"ff436c10e7220",[],"We also run a few VM images -- for Linux helper services, mostly -- and they are backed up through the same process. As our other host learned the hard way, backing up live VMs can be tricky, so this is ",{"_key":290,"_type":66,"marks":291,"text":292},"ff436c10e7221",[110],"definitely",{"_key":294,"_type":66,"marks":295,"text":296},"ff436c10e7222",[]," something you need to be careful about.",[],{"_key":299,"_type":62,"children":300,"level":116,"listItem":205,"markDefs":305,"style":91},"93ea85244b30",[301],{"_key":302,"_type":66,"marks":303,"text":304},"93ea85244b300",[],"We regularly download the latest database backups and restore them locally (we develop against live data all the time), so we know our backups work.",[],{"_key":307,"_type":62,"children":308,"markDefs":322,"style":91},"923b71998a52",[309,313,318],{"_key":310,"_type":66,"marks":311,"text":312},"923b71998a520",[],"This was originally documented in a private email, but I believe in ",{"_key":314,"_type":66,"marks":315,"text":317},"923b71998a521",[316],"1bafd1e3b965","maximizing the value of my keystrokes",{"_key":319,"_type":66,"marks":320,"text":321},"923b71998a522",[],", so I made it public. We try to be transparent in everything we do; hopefully this eases any lingering concerns over the blog outage.",[323],{"_key":316,"_type":89,"href":324,"reference":12},"http://www.codinghorror.com/blog/archives/000854.html",true,"2009/12/13",{"code":328,"language":329},"\u003Cp>On Friday, the server which hosts this blog suffered catastrophic data loss. Fortunately, the blog server is at \u003Cstrong>a different datacenter entirely\u003C/strong> than \u003Ca href=\"http://blog.stackoverflow.com/2009/02/new-datacenter-migration/\">PEAK, which hosts all the Trilogy sites\u003C/a>.\u003C/p>\n\u003Cp>It's a long story, and I'll document it in more detail elsewhere, but the short version is this:\u003C/p>\n\u003Cul>\u003Cli>\n\u003Cp>This particular host's (again, \u003Cem>not\u003C/em> PEAK)  backup processes were fatally flawed, as they were unable to backup a live virtual machine hard drive file. So the \"backups\" they had were nonexistent, because the backup process was failing on the most important server files every single night.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>We belatedly realized we should never have trusted the host with backups in the first place -- we should have been backing up the relevant bits of content in the virtual machine ourselves, instead of assuming the host's backups were working.\u003C/p>\n\u003C/li>\n\u003Cli>The blog was not a high priority for backup since it was a) on a singleton server in a completely different data center and b) it's not an actual trilogy site, but a \"helper\" site.\u003C/li>\n\u003C/ul>\u003Cp>Anyway, we were able to piece together a backup from different sources, and I took this opportunity to move the blog from WordPress on Windows (which has been \u003Cem>incredibly\u003C/em> quirky) to WordPress on Linux.\u003C/p>\n\u003Cp>You may be wondering how this incident relates to our \u003Cstrong>Stack Overflow disaster recovery plan\u003C/strong>. \u003C/p>\n\u003Cp>\u003Cimg src=\"http://stackoverflow.blog/wp-content/uploads/2017/02/our-disaster-recovery-plan.png\" alt=\"our-disaster-recovery-plan\">\u003C/p>\n\u003Cp>Mostly, it doesn't. I just never viewed the blog as part of our core mission (though I probably should have) and subsequently didn't give it the attention it deserved. We've since moved the blog to an actual PEAK \"family\" server in our rack, so it'll get folded in with our standard backup process.\u003C/p>\n\u003Cp>So what is our standard backup process?\u003C/p>\n\u003Col>\u003Cli>\n\u003Cp>We take full database backups of all databases at 4 AM, 4 PM, and 12 AM. (some databases are backed up more aggressively, but this is typical.) These full database backups are stored on \u003Ca href=\"http://blog.stackoverflow.com/2009/02/our-backup-strategy-inexpensive-nas/\">our NAS RAID-6 device\u003C/a> on the rack at the PEAK datacenter.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>We have a 500 GB USB hard drive attached to the database server. There is a C# script which copies the latest backups from the NAS to the USB hard drive every night at around 1 AM. The oldest files are deleted to make room for the new files as necessary. (The current Stack Overflow full backup is about 7 GB compressed, and the other databases are perhaps 2 GB compressed.) new: we'll have two USB hard drives connected and do identical copies in parallel in case one of the drives develops problems.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>One of our team members, \u003Ca href=\"http://blog.stackoverflow.com/2009/05/welcome-stack-overflow-valued-associate-00003/\">Geoff Dalgas\u003C/a>, lives a mile from the PEAK data center. He drops by and physically swaps out the USB hard drive every few weeks. He holds onefour 500 GB USB drives at his home, while the other two are at the data center. They continually get cycled back and forth over time.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>new: Fog Creek will FTP in and transfer the most current database backups to their hosting facility every week, during low traffic periods on Saturday.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>We do \u003Ca href=\"http://blog.stackoverflow.com/category/cc-wiki-dump/\">Creative Commons data dumps\u003C/a> of all sites (Stack Overflow, Server Fault, Super User) every month. This is a subset of the data, but a sizable one, and it's available on \u003Ca href=\"http://www.legaltorrents.com/creators/146-stack-overflow-data-dump\">Legal Torrents\u003C/a>. These data dumps are physically hosted on and seeded by Legal Torrents.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>Our Subversion source control repository is copied to the NAS every day and also gets copied to the USB external drive, etc, through the same script.\u003C/p>\n\u003C/li>\n\u003Cli>\n\u003Cp>We also run a few VM images -- for Linux helper services, mostly -- and they are backed up through the same process. As our other host learned the hard way, backing up live VMs can be tricky, so this is \u003Cem>definitely\u003C/em> something you need to be careful about.\u003C/p>\n\u003C/li>\n\u003Cli>We regularly download the latest database backups and restore them locally (we develop against live data all the time), so we know our backups work.\u003C/li>\n\u003C/ol>\u003Cp>This was originally documented in a private email, but I believe in \u003Ca href=\"http://www.codinghorror.com/blog/archives/000854.html\">maximizing the value of my keystrokes\u003C/a>, so I made it public. We try to be transparent in everything we do; hopefully this eases any lingering concerns over the blog outage.\u003C/p>","html","2009-12-13T12:00:00.000Z",{"current":332},"blog-outage-backup-policies",[334,342,347],{"_createdAt":335,"_id":336,"_rev":337,"_type":338,"_updatedAt":335,"slug":339,"title":341},"2023-05-23T16:43:21Z","wp-tagcat-background","9HpbCsT2tq0xwozQfkc4ih","blogTag",{"current":340},"background","Background",{"_createdAt":335,"_id":343,"_rev":337,"_type":338,"_updatedAt":335,"slug":344,"title":346},"wp-tagcat-company",{"current":345},"company","Company",{"_createdAt":335,"_id":348,"_rev":337,"_type":338,"_updatedAt":335,"slug":349,"title":351},"wp-tagcat-server",{"current":350},"server","Server","Blog Outage - Backup Policies",[354,360,366,372],{"_id":355,"publishedAt":356,"slug":357,"sponsored":12,"title":359},"28e560af-f0aa-4d46-bd90-f435ad604aa7","2026-06-26T14:00:27.102Z",{"_type":10,"current":358},"paging-charity-how-can-engineering-leaders-avoid-becoming-bond-villains","Paging Charity! How can engineering leaders avoid becoming Bond villains?",{"_id":361,"publishedAt":362,"slug":363,"sponsored":12,"title":365},"4b22c2a3-3779-4966-93eb-5230391dbdce","2026-06-23T14:08:58.595Z",{"_type":10,"current":364},"your-ai-shipped-a-backend-that-boots-that-is-the-whole-problem","Your AI shipped a backend that boots. That is the whole problem.",{"_id":367,"publishedAt":368,"slug":369,"sponsored":12,"title":371},"5cf362e1-fe7b-45af-b69c-914731c6a052","2026-06-23T14:00:00.000Z",{"_type":10,"current":370},"the-2026-developer-survey-is-now-open-for-human-developers-only","The 2026 Developer Survey is now open (for human developers only)!",{"_id":373,"publishedAt":374,"slug":375,"sponsored":12,"title":377},"30b995f7-7cb9-4dd8-bf71-d0685940a32b","2026-06-19T14:00:00.000Z",{"_type":10,"current":376},"dispatches-from-o-reilly-from-capabilities-to-responsibilities","Dispatches from O'Reilly: From capabilities to responsibilities",{"data":379,"sourceMap":-1},{"count":380,"lastTimestamp":12},0]