[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"sanity-8WvFjaxo7jIua8dmO5KvHF5m8Sd02zFQmfENIVTuOiI":3,"sanity-BUGiRZ82KoJ1rXtTFR_O9wUOf0u5odUivP_QPWuNwSQ":410},{"data":4,"sourceMap":-1},{"latestPodcast":5,"latestReleases":14,"post":39,"recent":385},[6],{"_id":7,"publishedAt":8,"slug":9,"sponsored":12,"title":13},"50f4509c-4f55-4f11-8adc-5556e821ea77","2026-06-30T07:40:00.000Z",{"_type":10,"current":11},"slug","why-intent-prediction-needs-more-than-an-llm",null,"Why intent prediction needs more than an LLM",[15,21,27,33],{"_id":16,"publishedAt":17,"slug":18,"title":20},"eb5b66eb-9410-4329-83bb-22bbff39402a","2026-04-28T13:00:00.000Z",{"_type":10,"current":19},"turn-scattered-knowledge-into-trusted-intelligence","Turning scattered knowledge into trusted intelligence: Stack Internal 2026.3",{"_id":22,"publishedAt":23,"slug":24,"title":26},"369c2401-b62e-4a37-8ff8-bf603023ecad","2026-03-02T15:03:00.988Z",{"_type":10,"current":25},"what-s-new-at-stack-overflow-march-2026","What’s new at Stack Overflow: March 2026",{"_id":28,"publishedAt":29,"slug":30,"title":32},"5e9053a4-07ea-447c-91ea-29e0b6228537","2026-02-02T15:00:00.000Z",{"_type":10,"current":31},"what-s-new-at-stack-overflow-february-2026","What’s new at Stack Overflow: February 2026",{"_id":34,"publishedAt":35,"slug":36,"title":38},"a1b538eb-a8a6-46d0-80a1-ac70ec9bb935","2026-01-05T10:00:00.000-05:00",{"_type":10,"current":37},"what-s-new-at-stack-overflow-january-2026","What’s new at Stack Overflow: January 2026",{"_createdAt":40,"_id":41,"_rev":42,"_type":43,"_updatedAt":44,"author":45,"body":59,"comments":357,"dateUrl":358,"excerpt":89,"legacyBody":359,"product":12,"publishedAt":362,"slug":363,"sponsored":12,"tags":365,"title":384,"visible":357},"2023-05-25T09:36:57Z","wp-post-3484","dgl3SCUzppW3U2LvCoS1MC","blogPost","2023-07-13T14:54:25Z",[46],{"_createdAt":47,"_id":48,"_rev":49,"_type":50,"_updatedAt":51,"avatar":52,"employee":54,"name":55,"role":56,"slug":57},"2023-05-23T16:27:18Z","wp-author-114","07ZbrKPSUrjrV4wQ6fam8u","blogAuthor","2023-08-29T11:49:01Z",{"_type":53},"image","former","Jeff Atwood","Co-founder",{"current":58},"jeffatwood",[60,83,90,107,126,135,162,190,224,243,273,292,300,330],{"_key":61,"_type":62,"children":63,"markDefs":78,"style":82},"75c3bfe849dc","block",[64,69,74],{"_key":65,"_type":66,"marks":67,"text":68},"75c3bfe849dc0","span",[],"We noticed something unusual on our ",{"_key":70,"_type":66,"marks":71,"text":73},"75c3bfe849dc1",[72],"986cc464f506","Cacti",{"_key":75,"_type":66,"marks":76,"text":77},"75c3bfe849dc2",[]," graphs today. Can you spot it?",[79],{"_key":72,"_type":80,"href":81,"reference":12},"link","http:\u002F\u002Fwww.cacti.net\u002F","normal",{"_key":84,"_type":53,"alt":85,"asset":86,"caption":89,"markDefs":12},"58aaa846bed9","stackoverflow-cacti-graph-june-15-2009",{"_ref":87,"_type":88},"image-926f171d45910c4ca0ba8bc7aa562e74c5843cbf-603x286-png","reference","",{"_key":91,"_type":62,"children":92,"markDefs":106,"style":82},"a13c48c297dd",[93,97,102],{"_key":94,"_type":66,"marks":95,"text":96},"a13c48c297dd0",[],"Yes! The light gray of the graph background ",{"_key":98,"_type":66,"marks":99,"text":101},"a13c48c297dd1",[100],"em","does",{"_key":103,"_type":66,"marks":104,"text":105},"a13c48c297dd2",[]," seem a few shades lighter than normal! I see it too!",[],{"_key":108,"_type":62,"children":109,"markDefs":123,"style":82},"c178e47d97e8",[110,114,119],{"_key":111,"_type":66,"marks":112,"text":113},"c178e47d97e80",[],"No, no, of course I'm talking about that massive traffic spike from 06:00 to 15:00 PST (server time). In ",{"_key":115,"_type":66,"marks":116,"text":118},"c178e47d97e81",[117],"ad7a0c0dc98c","the words of The Office's David Brent",{"_key":120,"_type":66,"marks":121,"text":122},"c178e47d97e82",[],":",[124],{"_key":117,"_type":80,"href":125,"reference":12},"http:\u002F\u002Fwww.youtube.com\u002Fwatch?v=3nyCIqJP_4s",{"_key":127,"_type":62,"children":128,"markDefs":133,"style":134},"1f50cd608888",[129],{"_key":130,"_type":66,"marks":131,"text":132},"1f50cd6088880",[],"I think there's been a rape up there!",[],"blockquote",{"_key":136,"_type":62,"children":137,"markDefs":159,"style":82},"b1ff45bed3b0",[138,142,147,151,155],{"_key":139,"_type":66,"marks":140,"text":141},"b1ff45bed3b00",[],"Bandwidth isn't usually a problem for us, as we are heavily text-oriented and go to great lengths to ",{"_key":143,"_type":66,"marks":144,"text":146},"b1ff45bed3b01",[145],"5f0f3b70acff","make sure all our text content is served up compressed",{"_key":148,"_type":66,"marks":149,"text":150},"b1ff45bed3b02",[],". This is almost 3x our normal ",{"_key":152,"_type":66,"marks":153,"text":154},"b1ff45bed3b03",[100],"peak",{"_key":156,"_type":66,"marks":157,"text":158},"b1ff45bed3b04",[]," traffic level. And for what?",[160],{"_key":145,"_type":80,"href":161,"reference":12},"http:\u002F\u002Fblog.stackoverflow.com\u002F2009\u002F02\u002Fhappy-100000th-question\u002F",{"_key":163,"_type":62,"children":164,"markDefs":187,"style":82},"0abc6a6ebbbc",[165,169,174,178,183],{"_key":166,"_type":66,"marks":167,"text":168},"0abc6a6ebbbc0",[],"Geoff ran a few queries in ",{"_key":170,"_type":66,"marks":171,"text":173},"0abc6a6ebbbc1",[172],"00a7e27da65b","Log Parser",{"_key":175,"_type":66,"marks":176,"text":177},"0abc6a6ebbbc2",[]," and found that this is yet another instance of ",{"_key":179,"_type":66,"marks":180,"text":182},"0abc6a6ebbbc3",[181],"strong","a perfect web spider storm",{"_key":184,"_type":66,"marks":185,"text":186},"0abc6a6ebbbc4",[],". Here are the top 3 bandwidth consumers in the logs for that day:",[188],{"_key":172,"_type":80,"href":189,"reference":12},"http:\u002F\u002Fwww.microsoft.com\u002Fdownloads\u002Fdetails.aspx?FamilyID=890cd06b-abf8-4c25-91b2-f8d975cf8c07&displaylang=en",{"_key":191,"_type":62,"children":192,"markDefs":219,"style":82},"a2e826714a66",[193,197,201,206,210,215],{"_key":194,"_type":66,"marks":195,"text":196},"a2e826714a660",[181],"IPUser-AgentRequestsBytes Served",{"_key":198,"_type":66,"marks":199,"text":200},"a2e826714a661",[],"72.30.78.240 ",{"_key":202,"_type":66,"marks":203,"text":205},"a2e826714a662",[204],"685bd080136e","Yahoo! Slurp\u002F3.0",{"_key":207,"_type":66,"marks":208,"text":209},"a2e826714a663",[],"56,331 1,124,861,78066.249.68.109 ",{"_key":211,"_type":66,"marks":212,"text":214},"a2e826714a664",[213],"060ac391add3","Googlebot\u002F2.1",{"_key":216,"_type":66,"marks":217,"text":218},"a2e826714a665",[],"56,579 773,418,83466.249.68.109 Mediapartners-Google 30,519 671,904,609",[220,222],{"_key":204,"_type":80,"href":221,"reference":12},"http:\u002F\u002Fhelp.yahoo.com\u002Fhelp\u002Fus\u002Fysearch\u002Fslurp",{"_key":213,"_type":80,"href":223,"reference":12},"http:\u002F\u002Fwww.google.com\u002Fbot.html",{"_key":225,"_type":62,"children":226,"markDefs":240,"style":82},"3d6f312a474e",[227,231,236],{"_key":228,"_type":66,"marks":229,"text":230},"3d6f312a474e0",[],"As I mentioned, this has happened to us before -- and we've considered ",{"_key":232,"_type":66,"marks":233,"text":235},"3d6f312a474e1",[234],"02a884d08732","dynamically blocking excessive HTTP bandwidth use",{"_key":237,"_type":66,"marks":238,"text":239},"3d6f312a474e2",[],". But first we politely asked the Yahoo and Google web spider bots to play a bit nicer:",[241],{"_key":234,"_type":80,"href":242,"reference":12},"http:\u002F\u002Fserverfault.com\u002Fquestions\u002F20383\u002Fdynamically-blocking-excessive-http-bandwith-use",{"_key":244,"_type":62,"children":245,"level":268,"listItem":269,"markDefs":270,"style":82},"c42cda817688",[246,250,255,259,264],{"_key":247,"_type":66,"marks":248,"text":249},"c42cda8176880",[],"We updated our ",{"_key":251,"_type":66,"marks":252,"text":254},"c42cda8176881",[253],"7982d06bfb8e","robots.txt",{"_key":256,"_type":66,"marks":257,"text":258},"c42cda8176882",[]," to include the ",{"_key":260,"_type":66,"marks":261,"text":263},"c42cda8176883",[262],"code","Crawl-delay",{"_key":265,"_type":66,"marks":266,"text":267},"c42cda8176884",[]," directive, like so: User-agent: Slurp Crawl-delay: 1 User-agent: msnbot Crawl-delay: 1",1,"number",[271],{"_key":253,"_type":80,"href":272,"reference":12},"http:\u002F\u002Fstackoverflow.com\u002Frobots.txt",{"_key":274,"_type":62,"children":275,"level":268,"listItem":269,"markDefs":289,"style":82},"8723e1ed7dda",[276,280,285],{"_key":277,"_type":66,"marks":278,"text":279},"8723e1ed7dda0",[],"We went to ",{"_key":281,"_type":66,"marks":282,"text":284},"8723e1ed7dda1",[283],"a30afd6b3e18","Google Webmaster Tools",{"_key":286,"_type":66,"marks":287,"text":288},"8723e1ed7dda2",[]," and told Google to send no more than 2 Googlebot search engine spider requests per second.",[290],{"_key":283,"_type":80,"href":291,"reference":12},"https:\u002F\u002Fwww.google.com\u002Fwebmasters\u002Ftools\u002F",{"_key":293,"_type":62,"children":294,"markDefs":299,"style":82},"f74ec0eb28e6",[295],{"_key":296,"_type":66,"marks":297,"text":298},"f74ec0eb28e60",[],"That was a week ago. Obviously, it isn't working.",[],{"_key":301,"_type":62,"children":302,"markDefs":325,"style":82},"2bdbde145192",[303,307,312,316,321],{"_key":304,"_type":66,"marks":305,"text":306},"2bdbde1451920",[],"Now we'll have to do this the hard way. Fortunately, Geoff (aka ",{"_key":308,"_type":66,"marks":309,"text":311},"2bdbde1451921",[310],"436f64541c57","Valued Stack Overflow Associate #00003",{"_key":313,"_type":66,"marks":314,"text":315},"2bdbde1451922",[],") has a \"spare\" ",{"_key":317,"_type":66,"marks":318,"text":320},"2bdbde1451923",[319],"b7b13f77bd18","Cisco PIX 515E",{"_key":322,"_type":66,"marks":323,"text":324},"2bdbde1451924",[]," laying around that we plan to put in front of the web servers, so we can dynamically throttle the offenders. But we can't do that for a week or so.",[326,328],{"_key":310,"_type":80,"href":327,"reference":12},"http:\u002F\u002Fblog.stackoverflow.com\u002F2009\u002F05\u002Fwelcome-stack-overflow-valued-associate-00003\u002F",{"_key":319,"_type":80,"href":329,"reference":12},"http:\u002F\u002Fwww.cisco.com\u002Fen\u002FUS\u002Fproducts\u002Fhw\u002Fvpndevc\u002Fps2030\u002Fps4094\u002F",{"_key":331,"_type":62,"children":332,"markDefs":354,"style":82},"ecc622e53a17",[333,337,341,345,350],{"_key":334,"_type":66,"marks":335,"text":336},"ecc622e53a170",[],"In the meantime, since Yahoo (via Slurp!) is about 0.3% of our traffic, but insists on rudely consuming a huge chunk of our prime-time bandwidth, ",{"_key":338,"_type":66,"marks":339,"text":340},"ecc622e53a171",[181],"they're getting IP banned and blocked",{"_key":342,"_type":66,"marks":343,"text":344},"ecc622e53a172",[],". I'm a bit more sympathetic to Google, since ",{"_key":346,"_type":66,"marks":347,"text":349},"ecc622e53a173",[348],"c3be8643ed2a","they deliver almost 90% of our traffic",{"_key":351,"_type":66,"marks":352,"text":353},"ecc622e53a174",[],", but it sure would be nice if they'd allow me to at least schedule the massive web spider storms for off-peak hours...",[355],{"_key":348,"_type":80,"href":356,"reference":12},"http:\u002F\u002Fwww.codinghorror.com\u002Fblog\u002Farchives\u002F001224.html",true,"2009\u002F06\u002F16",{"code":360,"language":361},"\u003Cp>We noticed something unusual on our \u003Ca href=\"http:\u002F\u002Fwww.cacti.net\u002F\">Cacti\u003C\u002Fa> graphs today. Can you spot it?\u003C\u002Fp>\n\u003Cp>\u003Cimg src=\"http:\u002F\u002Fstackoverflow.blog\u002Fwp-content\u002Fuploads\u002F2017\u002F02\u002Fstackoverflow-cacti-graph-june-15-2009.png\" alt=\"stackoverflow-cacti-graph-june-15-2009\">\u003C\u002Fp>\n\u003Cp>Yes! The light gray of the graph background \u003Cem>does\u003C\u002Fem> seem a few shades lighter than normal! I see it too!\u003C\u002Fp>\n\u003Cp>No, no, of course I'm talking about that massive traffic spike from 06:00 to 15:00 PST (server time). In \u003Ca href=\"http:\u002F\u002Fwww.youtube.com\u002Fwatch?v=3nyCIqJP_4s\">the words of The Office's David Brent\u003C\u002Fa>:\u003C\u002Fp>\n\u003Cblockquote>\nI think there's been a rape up there!\n\u003C\u002Fblockquote>\n\u003Cp>Bandwidth isn't usually a problem for us, as we are heavily text-oriented and go to great lengths to \u003Ca href=\"http:\u002F\u002Fblog.stackoverflow.com\u002F2009\u002F02\u002Fhappy-100000th-question\u002F\">make sure all our text content is served up compressed\u003C\u002Fa>. This is almost 3x our normal \u003Cem>peak\u003C\u002Fem> traffic level. And for what?\u003C\u002Fp>\n\u003Cp>Geoff ran a few queries in \u003Ca href=\"http:\u002F\u002Fwww.microsoft.com\u002Fdownloads\u002Fdetails.aspx?FamilyID=890cd06b-abf8-4c25-91b2-f8d975cf8c07&displaylang=en\">Log Parser\u003C\u002Fa> and found that this is yet another instance of \u003Cstrong>a perfect web spider storm\u003C\u002Fstrong>. Here are the top 3 bandwidth consumers in the logs for that day:\u003C\u002Fp>\n\u003Ctable width=\"600\">\u003Ctr>\u003Ctd>\u003Cstrong>IP\u003C\u002Fstrong>\n\u003C\u002Ftd>\u003Ctd>\u003Cstrong>User-Agent\u003C\u002Fstrong>\n\u003C\u002Ftd>\u003Ctd align=\"right\">\u003Cstrong>Requests\u003C\u002Fstrong>\n\u003C\u002Ftd>\u003Ctd align=\"right\">\u003Cstrong>Bytes Served\u003C\u002Fstrong>\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd>72.30.78.240\n\u003C\u002Ftd>\u003Ctd>\u003Ca href=\"http:\u002F\u002Fhelp.yahoo.com\u002Fhelp\u002Fus\u002Fysearch\u002Fslurp\">Yahoo! Slurp\u002F3.0\u003C\u002Fa>\n\u003C\u002Ftd>\u003Ctd align=\"right\">56,331\n\u003C\u002Ftd>\u003Ctd align=\"right\">1,124,861,780\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd>66.249.68.109\n\u003C\u002Ftd>\u003Ctd>\u003Ca href=\"http:\u002F\u002Fwww.google.com\u002Fbot.html\">Googlebot\u002F2.1\u003C\u002Fa>\n\u003C\u002Ftd>\u003Ctd align=\"right\">56,579\n\u003C\u002Ftd>\u003Ctd align=\"right\">773,418,834\u003C\u002Ftd>\u003C\u002Ftr>\u003Ctr>\u003Ctd>66.249.68.109\n\u003C\u002Ftd>\u003Ctd>Mediapartners-Google\n\u003C\u002Ftd>\u003Ctd align=\"right\">30,519\n\u003C\u002Ftd>\u003Ctd align=\"right\">671,904,609\u003C\u002Ftd>\u003C\u002Ftr>\u003C\u002Ftable>\u003Cp>As I mentioned, this has happened to us before -- and we've considered \u003Ca href=\"http:\u002F\u002Fserverfault.com\u002Fquestions\u002F20383\u002Fdynamically-blocking-excessive-http-bandwith-use\">dynamically blocking excessive HTTP bandwidth use\u003C\u002Fa>. But first we politely asked the Yahoo and Google web spider bots to play a bit nicer:\u003C\u002Fp>\n\u003Col>\u003Cli>\n\u003Cp>We updated our \u003Ca href=\"http:\u002F\u002Fstackoverflow.com\u002Frobots.txt\">robots.txt\u003C\u002Fa> to include the \u003Ccode>Crawl-delay\u003C\u002Fcode> directive, like so:\u003C\u002Fp>\n\u003Cp>User-agent: Slurp\nCrawl-delay: 1\u003C\u002Fp>\n\u003Cp>User-agent: msnbot\nCrawl-delay: 1\u003C\u002Fp>\n\u003C\u002Fli>\n\u003Cli>We went to \u003Ca href=\"https:\u002F\u002Fwww.google.com\u002Fwebmasters\u002Ftools\u002F\">Google Webmaster Tools\u003C\u002Fa> and told Google to send no more than 2 Googlebot search engine spider requests per second.\u003C\u002Fli>\n\u003C\u002Fol>\u003Cp>That was a week ago. Obviously, it isn't working.\u003C\u002Fp>\n\u003Cp>Now we'll have to do this the hard way. Fortunately, Geoff (aka \u003Ca href=\"http:\u002F\u002Fblog.stackoverflow.com\u002F2009\u002F05\u002Fwelcome-stack-overflow-valued-associate-00003\u002F\">Valued Stack Overflow Associate #00003\u003C\u002Fa>) has a \"spare\" \u003Ca href=\"http:\u002F\u002Fwww.cisco.com\u002Fen\u002FUS\u002Fproducts\u002Fhw\u002Fvpndevc\u002Fps2030\u002Fps4094\u002F\">Cisco PIX 515E\u003C\u002Fa> laying around that we plan to put in front of the web servers, so we can dynamically throttle the offenders. But we can't do that for a week or so. \u003C\u002Fp>\n\u003Cp>In the meantime, since Yahoo (via Slurp!) is about 0.3% of our traffic, but insists on rudely consuming a huge chunk of our prime-time bandwidth, \u003Cstrong>they're getting IP banned and blocked\u003C\u002Fstrong>. I'm a bit more sympathetic to Google, since \u003Ca href=\"http:\u002F\u002Fwww.codinghorror.com\u002Fblog\u002Farchives\u002F001224.html\">they deliver almost 90% of our traffic\u003C\u002Fa>, but it sure would be nice if they'd allow me to at least schedule the massive web spider storms for off-peak hours...\u003C\u002Fp>","html","2009-06-16T12:00:00.000Z",{"current":364},"the-perfect-web-spider-storm",[366,374,379],{"_createdAt":367,"_id":368,"_rev":369,"_type":370,"_updatedAt":367,"slug":371,"title":373},"2023-05-23T16:43:21Z","wp-tagcat-background","9HpbCsT2tq0xwozQfkc4ih","blogTag",{"current":372},"background","Background",{"_createdAt":367,"_id":375,"_rev":369,"_type":370,"_updatedAt":367,"slug":376,"title":378},"wp-tagcat-company",{"current":377},"company","Company",{"_createdAt":367,"_id":380,"_rev":369,"_type":370,"_updatedAt":367,"slug":381,"title":383},"wp-tagcat-server",{"current":382},"server","Server","The Perfect Web Spider Storm",[386,392,398,404],{"_id":387,"publishedAt":388,"slug":389,"sponsored":12,"title":391},"28e560af-f0aa-4d46-bd90-f435ad604aa7","2026-06-26T14:00:27.102Z",{"_type":10,"current":390},"paging-charity-how-can-engineering-leaders-avoid-becoming-bond-villains","Paging Charity! How can engineering leaders avoid becoming Bond villains?",{"_id":393,"publishedAt":394,"slug":395,"sponsored":12,"title":397},"4b22c2a3-3779-4966-93eb-5230391dbdce","2026-06-23T14:08:58.595Z",{"_type":10,"current":396},"your-ai-shipped-a-backend-that-boots-that-is-the-whole-problem","Your AI shipped a backend that boots. That is the whole problem.",{"_id":399,"publishedAt":400,"slug":401,"sponsored":12,"title":403},"5cf362e1-fe7b-45af-b69c-914731c6a052","2026-06-23T14:00:00.000Z",{"_type":10,"current":402},"the-2026-developer-survey-is-now-open-for-human-developers-only","The 2026 Developer Survey is now open (for human developers only)!",{"_id":405,"publishedAt":406,"slug":407,"sponsored":12,"title":409},"30b995f7-7cb9-4dd8-bf71-d0685940a32b","2026-06-19T14:00:00.000Z",{"_type":10,"current":408},"dispatches-from-o-reilly-from-capabilities-to-responsibilities","Dispatches from O'Reilly: From capabilities to responsibilities",{"data":411,"sourceMap":-1},{"count":412,"lastTimestamp":12},0]