[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"sanity-_0jxSRZ4tHE6bsnNik4lx8MeXT-SeLpsGK_hPSe3zY4":3,"sanity-Pse-_D5DgjrixZBnwOVWUxdgt2qXWX10lo7KnB9ydnY":529},{"data":4,"sourceMap":-1},{"latestPodcast":5,"latestReleases":14,"post":39,"recent":504},[6],{"_id":7,"publishedAt":8,"slug":9,"sponsored":12,"title":13},"4d0175f4-40a8-47eb-9bb3-a453b326aa7d","2026-07-03T07:40:00.000Z",{"_type":10,"current":11},"slug","the-good-the-bad-and-the-ai-apps",null,"The good, the bad, and the AI apps",[15,21,27,33],{"_id":16,"publishedAt":17,"slug":18,"title":20},"eb5b66eb-9410-4329-83bb-22bbff39402a","2026-04-28T13:00:00.000Z",{"_type":10,"current":19},"turn-scattered-knowledge-into-trusted-intelligence","Turning scattered knowledge into trusted intelligence: Stack Internal 2026.3",{"_id":22,"publishedAt":23,"slug":24,"title":26},"369c2401-b62e-4a37-8ff8-bf603023ecad","2026-03-02T15:03:00.988Z",{"_type":10,"current":25},"what-s-new-at-stack-overflow-march-2026","What’s new at Stack Overflow: March 2026",{"_id":28,"publishedAt":29,"slug":30,"title":32},"5e9053a4-07ea-447c-91ea-29e0b6228537","2026-02-02T15:00:00.000Z",{"_type":10,"current":31},"what-s-new-at-stack-overflow-february-2026","What’s new at Stack Overflow: February 2026",{"_id":34,"publishedAt":35,"slug":36,"title":38},"a1b538eb-a8a6-46d0-80a1-ac70ec9bb935","2026-01-05T10:00:00.000-05:00",{"_type":10,"current":37},"what-s-new-at-stack-overflow-january-2026","What’s new at Stack Overflow: January 2026",{"_createdAt":40,"_id":41,"_rev":42,"_type":43,"_updatedAt":44,"author":45,"body":73,"comments":462,"dateUrl":463,"excerpt":464,"image":465,"legacyBody":468,"product":12,"publishedAt":471,"slug":472,"sponsored":12,"tags":474,"title":503,"visible":462},"2023-05-25T09:39:11Z","wp-post-9530","07ZbrKPSUrjrV4wQ6gvDbf","blogPost","2023-08-30T13:20:48Z",[46,62],{"_createdAt":47,"_id":48,"_rev":49,"_type":50,"_updatedAt":51,"avatar":52,"employee":57,"name":58,"role":59,"slug":60},"2023-05-23T16:27:18Z","wp-author-113","dgl3SCUzppW3U2LvCq2BPw","blogAuthor","2023-08-30T13:19:31Z",{"_type":53,"asset":54},"image",{"_ref":55,"_type":56},"image-b16d0cd9c04e0f49509605090eed45c60ec7d920-1024x1024-png","reference","former","Jason Punyon","Data Science Engineer",{"current":61},"jasonpunyon",{"_createdAt":47,"_id":63,"_rev":64,"_type":50,"_updatedAt":65,"avatar":66,"employee":57,"name":69,"role":70,"slug":71},"wp-author-125","9HpbCsT2tq0xwozQflrfOh","2023-08-30T13:15:26Z",{"_type":53,"asset":67},{"_ref":68,"_type":56},"image-8c4101ec5a80bd817bd18a920b025b97dff07164-1022x1024-jpg","Julia Silge","Data Scientist",{"current":72},"juliasilge",[74,97,106,135,152,171,190,209,213,230,238,267,284,292,300,308,324,332,340,348,352,381,385,423,427,446,454],{"_key":75,"_type":76,"children":77,"markDefs":92,"style":96},"eeda3e7d29f0","block",[78,83,88],{"_key":79,"_type":80,"marks":81,"text":82},"eeda3e7d29f00","span",[],"This past summer, we wrote our ",{"_key":84,"_type":80,"marks":85,"text":87},"eeda3e7d29f01",[86],"3b4ea18fae99","first blog post about comments on Stack Overflow",{"_key":89,"_type":80,"marks":90,"text":91},"eeda3e7d29f02",[],", focusing on our initial work rating comments internally at Stack Overflow and what we learned. Since then, we've fielded this comment rating task more broadly in our community. This blog post shares some of what we are learning.",[93],{"_key":86,"_type":94,"href":95,"reference":12},"link","https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F07\u002F10\u002Fwelcome-wagon-classifying-comments-on-stack-overflow\u002F","normal",{"_key":98,"_type":76,"children":99,"markDefs":104,"style":105},"e2567249c15f",[100],{"_key":101,"_type":80,"marks":102,"text":103},"e2567249c15f0",[],"Engaging our community",[],"h2",{"_key":107,"_type":76,"children":108,"markDefs":131,"style":96},"fe8e21c5670d",[109,113,118,122,127],{"_key":110,"_type":80,"marks":111,"text":112},"fe8e21c5670d0",[],"I (Jason) wrote a web application that presents a user with a comment thread from a post on Stack Overflow and asks the user to rate each comment in the thread as fine, unwelcoming, or abusive. Our ",{"_key":114,"_type":80,"marks":115,"text":117},"fe8e21c5670d1",[116],"ea7db07bb9cd","first blog post",{"_key":119,"_type":80,"marks":120,"text":121},"fe8e21c5670d2",[]," shared results from when we asked employees at Stack Overflow, including developers, product managers, and executives, to rate comments. In August, we rolled out our new ",{"_key":123,"_type":80,"marks":124,"text":126},"fe8e21c5670d3",[125],"5d8e0110bea3","Code of Conduct",{"_key":128,"_type":80,"marks":129,"text":130},"fe8e21c5670d4",[],", along with new flags for comments that align with these categories, one flag for rude\u002Fabusive and one flag for unfriendly\u002Funkind. This fall, we extended our comment classification task beyond our employees to our larger community. We invited individuals from three groups to rate comments.",[132,133],{"_key":116,"_type":94,"href":95,"reference":12},{"_key":125,"_type":94,"href":134,"reference":12},"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F08\u002F07\u002Fget-to-know-our-new-code-of-conduct\u002F",{"_key":136,"_type":76,"children":137,"level":147,"listItem":148,"markDefs":149,"style":96},"cd85d85ce22e",[138,143],{"_key":139,"_type":80,"marks":140,"text":142},"cd85d85ce22e0",[141],"3b8049dc4848","Moderators",{"_key":144,"_type":80,"marks":145,"text":146},"cd85d85ce22e1",[]," on Stack Overflow and other Stack Exchange sites",1,"bullet",[150],{"_key":141,"_type":94,"href":151,"reference":12},"https:\u002F\u002Fstackoverflow.com\u002Fhelp\u002Fsite-moderators",{"_key":153,"_type":76,"children":154,"level":147,"listItem":148,"markDefs":168,"style":96},"8bc09c360b3e",[155,159,164],{"_key":156,"_type":80,"marks":157,"text":158},"8bc09c360b3e0",[],"Individuals who responded to ",{"_key":160,"_type":80,"marks":161,"text":163},"8bc09c360b3e1",[162],"31329ae3e25c","our blog post in April",{"_key":165,"_type":80,"marks":166,"text":167},"8bc09c360b3e2",[],", indicating they want to help make Stack Overflow more welcoming",[169],{"_key":162,"_type":94,"href":170,"reference":12},"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F04\u002F26\u002Fstack-overflow-isnt-very-welcoming-its-time-for-that-to-change\u002F",{"_key":172,"_type":76,"children":173,"level":147,"listItem":148,"markDefs":187,"style":96},"42b12e3160de",[174,178,183],{"_key":175,"_type":80,"marks":176,"text":177},"42b12e3160de0",[],"A sample of registered users from our general research list (you can opt in\u002Fout of our research list in ",{"_key":179,"_type":80,"marks":180,"text":182},"42b12e3160de1",[181],"cc773713277a","your Stack Overflow email settings",{"_key":184,"_type":80,"marks":185,"text":186},"42b12e3160de2",[],")",[188],{"_key":181,"_type":94,"href":189,"reference":12},"https:\u002F\u002Fstackoverflow.com\u002Fusers\u002Femail\u002Fsettings\u002Fcurrent",{"_key":191,"_type":76,"children":192,"markDefs":206,"style":96},"37c863ddc882",[193,197,202],{"_key":194,"_type":80,"marks":195,"text":196},"37c863ddc8820",[],"To log in to this web app and record data, each user needed a Stack Overflow account, so users had to make an account if they didn't have one already. We asked participants to invest at least one hour in rating comments, and to not work for more than 20 minutes at one sitting. What kind of response did we get? Overall, there were 525 users who spent at least 15 minutes or more rating comments. They made 253,807 ratings of 40,358 distinct comments. How many users and comment ratings did we have, for each kind of user? [table id=4 \u002F] The moderators demonstrated their enormous commitment to our community through this project, as they do consistently day in, day out; moderators who participated in this project rated an average of over 1,000 comments each. Folks who responded to our blog post expressing interest in welcome\u002Finclusion on Stack Overflow also invested a great deal of time, rating over 500 comments each. We see can see this visually by looking at the ",{"_key":198,"_type":80,"marks":199,"text":201},"37c863ddc8821",[200],"9e048ecf248a","cumulative distribution functions",{"_key":203,"_type":80,"marks":204,"text":205},"37c863ddc8822",[]," for each kind of user; this kind of plot shows, for each number of comment ratings, the percentage of users who rated that many comments or lower.",[207],{"_key":200,"_type":94,"href":208,"reference":12},"https:\u002F\u002Fmath.stackexchange.com\u002Fquestions\u002F52400\u002Fwhat-is-cdf-cumulative-distribution-function\u002F52416",{"_key":210,"_type":53,"alt":12,"asset":211,"markDefs":12},"bb12d529626e",{"_ref":212,"_type":56},"image-831678d85acebfcda091ae4b9b5dbeeee3e5890d-945x675-png",{"_key":214,"_type":76,"children":215,"markDefs":229,"style":96},"58565828106f",[216,220,225],{"_key":217,"_type":80,"marks":218,"text":219},"58565828106f0",[],"If you're not used to interpreting this kind of graph, take a look at ",{"_key":221,"_type":80,"marks":222,"text":224},"58565828106f1",[223],"code","x = 1000",{"_key":226,"_type":80,"marks":227,"text":228},"58565828106f2",[],", the location on the x-axis that corresponds to 1,000 comments. The line for the moderators is the lowest, indicating that more moderators submitted more comment ratings compared to the other groups.",[],{"_key":231,"_type":76,"children":232,"markDefs":237,"style":105},"e4833feb9aa6",[233],{"_key":234,"_type":80,"marks":235,"text":236},"e4833feb9aa60",[],"Group differences",[],{"_key":239,"_type":76,"children":240,"markDefs":263,"style":96},"8f06574c18a2",[241,245,250,254,259],{"_key":242,"_type":80,"marks":243,"text":244},"8f06574c18a20",[],"Different kinds of people experience Stack Overflow in different ways. If we look at all the ratings made by the different types of users aggregated, how do the different kinds of groups perceive these comments on Stack Overflow? [table id=5 \u002F] The highest rates of unwelcoming comments were identified by the internal employees at Stack Overflow, followed by Stack Exchange moderators. We ",{"_key":246,"_type":80,"marks":247,"text":249},"8f06574c18a21",[248],"1f63c741e554","trust and support our moderators",{"_key":251,"_type":80,"marks":252,"text":253},"8f06574c18a22",[],", and in this specific project, moderators demonstrated their understanding of unfriendly and unwelcoming behavior in comments. Regular registered users from our research list perceived the next lowest rate of unfriendly comments, and users who responded to our blog post about ",{"_key":255,"_type":80,"marks":256,"text":258},"8f06574c18a23",[257],"e24e55042f34","making Stack Overflow more welcoming",{"_key":260,"_type":80,"marks":261,"text":262},"8f06574c18a24",[]," found the lowest rates of unfriendly comments of all. How can we interpret this? We specifically invited users who may not consider themselves active participants in our community in order to gain outside perspective, but then these users saw the lowest rates of unwelcoming behavior. A possible explanation is that we are seeing a real effect of deep experience with our site; it appears the more invested an individual is here at Stack Overflow, the more sensitive they are to problematic behavior. What do these unfriendly comments look like? The following combine elements of real comments to show typical examples.",[264,266],{"_key":248,"_type":94,"href":265,"reference":12},"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F11\u002F21\u002Four-theory-of-moderation-re-visited\u002F",{"_key":257,"_type":94,"href":170,"reference":12},{"_key":268,"_type":76,"children":269,"level":147,"listItem":148,"markDefs":283,"style":96},"067116a7c858",[270,274,279],{"_key":271,"_type":80,"marks":272,"text":273},"067116a7c8580",[],"\"",{"_key":275,"_type":80,"marks":276,"text":278},"067116a7c8581",[277],"em","Why",{"_key":280,"_type":80,"marks":281,"text":282},"067116a7c8582",[]," do you want to do this? You have conflated at least three problems here.\"",[],{"_key":285,"_type":76,"children":286,"level":147,"listItem":148,"markDefs":291,"style":96},"ccb29b1eb597",[287],{"_key":288,"_type":80,"marks":289,"text":290},"ccb29b1eb5970",[],"\"It will be very hard to help you with such a trivial bug. It could come from any line in your code, and we have to guess.\"",[],{"_key":293,"_type":76,"children":294,"level":147,"listItem":148,"markDefs":299,"style":96},"f63221675c0b",[295],{"_key":296,"_type":80,"marks":297,"text":298},"f63221675c0b0",[],"\"How exactly is this going to solve my problem?!\"",[],{"_key":301,"_type":76,"children":302,"level":147,"listItem":148,"markDefs":307,"style":96},"e09c33b3f3eb",[303],{"_key":304,"_type":80,"marks":305,"text":306},"e09c33b3f3eb0",[],"\"You don't understand how to use this site. Here nobody codes for you; read the docs and then show us.\"",[],{"_key":309,"_type":76,"children":310,"level":147,"listItem":148,"markDefs":323,"style":96},"db52f4c3dbd4",[311,315,319],{"_key":312,"_type":80,"marks":313,"text":314},"db52f4c3dbd40",[],"\"What are you ",{"_key":316,"_type":80,"marks":317,"text":318},"db52f4c3dbd41",[277],"actually",{"_key":320,"_type":80,"marks":321,"text":322},"db52f4c3dbd42",[]," trying to achieve? Please learn how to use a debugger.\"",[],{"_key":325,"_type":76,"children":326,"markDefs":331,"style":96},"eb3367f8c024",[327],{"_key":328,"_type":80,"marks":329,"text":330},"eb3367f8c0240",[],"Our project showed that the more deeply an individual is connected to Stack Overflow (as an employee, or a moderator), the more they are likely to see problems in comments like these. This effect is robust to comparing groups who were shown the same comments, who rated the same number of comments, and other analytical approaches.",[],{"_key":333,"_type":76,"children":334,"markDefs":339,"style":105},"b173249abb9b",[335],{"_key":336,"_type":80,"marks":337,"text":338},"b173249abb9b0",[],"Individual differences",[],{"_key":341,"_type":76,"children":342,"markDefs":347,"style":96},"cc692bb75bf8",[343],{"_key":344,"_type":80,"marks":345,"text":346},"cc692bb75bf80",[],"What do the distributions of ratings for each individual look like?",[],{"_key":349,"_type":53,"alt":12,"asset":350,"markDefs":12},"e10b1d355156",{"_ref":351,"_type":56},"image-b0e04a54673f34a2d976741ef2dd9c23f9025ad2-1200x508-png",{"_key":353,"_type":76,"children":354,"markDefs":377,"style":96},"5c849dd7da22",[355,359,364,368,373],{"_key":356,"_type":80,"marks":357,"text":358},"5c849dd7da220",[],"Each individual did not rate the same set of comments and worked for different lengths of time, so we expect variability in the results for each individual. Overall, the median percentage of perceived unwelcoming comments per individual was 3.5%, quite a bit lower than the ",{"_key":360,"_type":80,"marks":361,"text":363},"5c849dd7da221",[362],"9a9c142abf70","median percentage for employees",{"_key":365,"_type":80,"marks":366,"text":367},"5c849dd7da222",[]," of 6.5%. To understand how much agreement there is between raters, we can again look at ",{"_key":369,"_type":80,"marks":370,"text":372},"5c849dd7da223",[371],"f9bf13dac729","Krippendorff's alpha",{"_key":374,"_type":80,"marks":375,"text":376},"5c849dd7da224",[],", a measure that ranges from zero (nobody agrees) to one (perfect agreement). This measure accounts for the number of raters, so we can compare agreement among employees to the groups with more raters. What is Krippendorff's alpha, for comments that were rated by at least three people?",[378,379],{"_key":362,"_type":94,"href":95,"reference":12},{"_key":371,"_type":94,"href":380,"reference":12},"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FKrippendorff%27s_alpha",{"_key":382,"_type":53,"alt":12,"asset":383,"markDefs":12},"c1a37a45d1df",{"_ref":384,"_type":56},"image-3b15b2a33b57a316697d2faf96a8783e7071b7aa-900x675-png",{"_key":386,"_type":76,"children":387,"markDefs":418,"style":96},"dcb5c36b3b98",[388,392,397,401,405,409,414],{"_key":389,"_type":80,"marks":390,"text":391},"dcb5c36b3b980",[],"These values for alpha are low compared to what social scientists would use to draw reliable conclusions based on the ratings; social scientists look for values close to 0.8 or more. Notice that Stack Overflow employees rated more comments as unwelcoming than other groups but agreed with each other about what is unwelcoming and abusive at higher rates at the same time. The rate of agreement among moderators and registered users was lowest (although still ",{"_key":393,"_type":80,"marks":394,"text":396},"dcb5c36b3b981",[395],"f9c33b297956","much higher than for people unfamiliar with Stack Overflow",{"_key":398,"_type":80,"marks":399,"text":400},"dcb5c36b3b982",[],"), and the rate of agreement for the users who volunteered to help make Stack Overflow more welcoming was a bit higher. Remember that these were the users who rated the ",{"_key":402,"_type":80,"marks":403,"text":404},"dcb5c36b3b983",[277],"lowest",{"_key":406,"_type":80,"marks":407,"text":408},"dcb5c36b3b984",[]," overall levels of unfriendliness; some spot-checking indicates these users identified only the clearest examples of problematic text. Another factor that impacts interactions on Stack Overflow is ",{"_key":410,"_type":80,"marks":411,"text":413},"dcb5c36b3b985",[412],"e494d0a7f840","reputation",{"_key":415,"_type":80,"marks":416,"text":417},"dcb5c36b3b986",[],". Do we see any difference in how raters perceived unwelcoming behavior with their own reputation? This can help us understand if \"power users\" (distinguished from moderators) may be driving problems with site culture.",[419,421],{"_key":395,"_type":94,"href":420,"reference":12},"https:\u002F\u002Fstats.stackexchange.com\u002Fquestions\u002F33566\u002Fis-joel-spolskys-hunting-of-the-snark-post-valid-statistical-content-analysis",{"_key":412,"_type":94,"href":422,"reference":12},"https:\u002F\u002Fstackoverflow.com\u002Fhelp\u002Fwhats-reputation",{"_key":424,"_type":53,"alt":12,"asset":425,"markDefs":12},"dd7cf2a4e05c",{"_ref":426,"_type":56},"image-d78eea376a99e983bf368ceeb08d3caacad1ee5b-1080x675-png",{"_key":428,"_type":76,"children":429,"markDefs":443,"style":96},"5e55f0e0285a",[430,434,439],{"_key":431,"_type":80,"marks":432,"text":433},"5e55f0e0285a0",[],"There is no clear evidence in this plot for a relationship, indicating that high-reputation users perceive unfriendly behavior at about the same rate as low-reputation users. We have seen effects similar to this before, for example, in our annual Developer Survey. When asked what the ",{"_key":435,"_type":80,"marks":436,"text":438},"5e55f0e0285a1",[437],"5da238028719","worst or most annoying thing about Stack Overflow is",{"_key":440,"_type":80,"marks":441,"text":442},"5e55f0e0285a2",[],", developers of all experience levels and self-reported activity levels on Stack Overflow mentioned issues with harsh interactions and site culture. All together, this begins to paint a complex and interesting picture of who understands unwelcoming behavior and in what ways. Moderators and high-reputation users are just as likely, or even more likely, to identify unwelcoming comments compared to new users. Stack Overflow employees identify more comments as problematic and agree with each other more about what is a problem compared to the other kinds of users in this project.",[444],{"_key":437,"_type":94,"href":445,"reference":12},"https:\u002F\u002Finsights.stackoverflow.com\u002Fsurvey\u002F2018#community-developers-perspectives-about-our-community",{"_key":447,"_type":76,"children":448,"markDefs":453,"style":105},"dfdf2628e63a",[449],{"_key":450,"_type":80,"marks":451,"text":452},"dfdf2628e63a0",[],"Next steps",[],{"_key":455,"_type":76,"children":456,"markDefs":461,"style":96},"19f50cfdd988",[457],{"_key":458,"_type":80,"marks":459,"text":460},"19f50cfdd9880",[],"So where do we go from here? For starters, we as employees learned that we don't always perceive problems in the same way as other members of our community. We will keep this in mind as we move forward with plans to make Stack Overflow a better place for developers to learn and share knowledge. We plan to use this dataset to investigate how comments are used on questions and answers, toward users of different experience levels, in different communities, and more. Look for more blog posts on these issues in upcoming months. We will continue to use the results from this project in product changes on Stack Overflow, as well as directly using appropriate subsets of this data in machine learning models. Also, in 2019 we will release this dataset (comment IDs, comment ratings, and anonymized\u002Frandomized rater ID) upon request so that other people in our community and beyond can explore this data for themselves. All of this would not be possible without the investment of time and energy of the individuals who participated in this project, and we want to acknowledge each of you who volunteered to help us understand this aspect of our site better. Thank you, for your care and time. Community is central to our identity at Stack Overflow, and we are committed to making Stack Overflow a healthy, inclusive place for developers to learn and share knowledge.",[],true,"2018\u002F12\u002F04","",{"_type":53,"asset":466},{"_ref":467,"_type":56},"image-31e69f658407f882ff4c7a53bdacb4c0ebdb19c8-1921x1081-png",{"code":469,"language":470},"This past summer, we wrote our \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F07\u002F10\u002Fwelcome-wagon-classifying-comments-on-stack-overflow\u002F\">first blog post about comments on Stack Overflow\u003C\u002Fa>, focusing on our initial work rating comments internally at Stack Overflow and what we learned. Since then, we've fielded this comment rating task more broadly in our community. This blog post shares some of what we are learning.\n\u003Ch2>Engaging our community\u003C\u002Fh2>\nI (Jason) wrote a web application that presents a user with a comment thread from a post on Stack Overflow and asks the user to rate each comment in the thread as fine, unwelcoming, or abusive. Our \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F07\u002F10\u002Fwelcome-wagon-classifying-comments-on-stack-overflow\u002F\">first blog post\u003C\u002Fa> shared results from when we asked employees at Stack Overflow, including developers, product managers, and executives, to rate comments. In August, we rolled out our new \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F08\u002F07\u002Fget-to-know-our-new-code-of-conduct\u002F\">Code of Conduct\u003C\u002Fa>, along with new flags for comments that align with these categories, one flag for rude\u002Fabusive and one flag for unfriendly\u002Funkind. This fall, we extended our comment classification task beyond our employees to our larger community. We invited individuals from three groups to rate comments.\n\u003Cul>\n \t\u003Cli>\u003Ca href=\"https:\u002F\u002Fstackoverflow.com\u002Fhelp\u002Fsite-moderators\">Moderators\u003C\u002Fa> on Stack Overflow and other Stack Exchange sites\u003C\u002Fli>\n \t\u003Cli>Individuals who responded to \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F04\u002F26\u002Fstack-overflow-isnt-very-welcoming-its-time-for-that-to-change\u002F\">our blog post in April\u003C\u002Fa>, indicating they want to help make Stack Overflow more welcoming\u003C\u002Fli>\n \t\u003Cli>A sample of registered users from our general research list (you can opt in\u002Fout of our research list in \u003Ca href=\"https:\u002F\u002Fstackoverflow.com\u002Fusers\u002Femail\u002Fsettings\u002Fcurrent\">your Stack Overflow email settings\u003C\u002Fa>)\u003C\u002Fli>\n\u003C\u002Ful>\nTo log in to this web app and record data, each user needed a Stack Overflow account, so users had to make an account if they didn't have one already. We asked participants to invest at least one hour in rating comments, and to not work for more than 20 minutes at one sitting.\n\nWhat kind of response did we get? Overall, there were 525 users who spent at least 15 minutes or more rating comments. They made 253,807 ratings of 40,358 distinct comments. How many users and comment ratings did we have, for each kind of user?\n\n[table id=4 \u002F]\n\nThe moderators demonstrated their enormous commitment to our community through this project, as they do consistently day in, day out; moderators who participated in this project rated an average of over 1,000 comments each. Folks who responded to our blog post expressing interest in welcome\u002Finclusion on Stack Overflow also invested a great deal of time, rating over 500 comments each.\n\nWe see can see this visually by looking at the \u003Ca href=\"https:\u002F\u002Fmath.stackexchange.com\u002Fquestions\u002F52400\u002Fwhat-is-cdf-cumulative-distribution-function\u002F52416\">cumulative distribution functions\u003C\u002Fa> for each kind of user; this kind of plot shows, for each number of comment ratings, the percentage of users who rated that many comments or lower.\n\n\u003Cimg class=\"aligncenter size-large wp-image-9535\" src=\"https:\u002F\u002Fstackoverflow.blog\u002Fwp-content\u002Fuploads\u002F2018\u002F11\u002Fcdf-1-945x675.png\" alt=\"\" width=\"945\" height=\"675\">\n\nIf you're not used to interpreting this kind of graph, take a look at \u003Ccode>x = 1000\u003C\u002Fcode>, the location on the x-axis that corresponds to 1,000 comments. The line for the moderators is the lowest, indicating that more moderators submitted more comment ratings compared to the other groups.\n\u003Ch2>Group differences\u003C\u002Fh2>\nDifferent kinds of people experience Stack Overflow in different ways. If we look at all the ratings made by the different types of users aggregated, how do the different kinds of groups perceive these comments on Stack Overflow?\n\n[table id=5 \u002F]\n\nThe highest rates of unwelcoming comments were identified by the internal employees at Stack Overflow, followed by Stack Exchange moderators. We \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F11\u002F21\u002Four-theory-of-moderation-re-visited\u002F\">trust and support our moderators\u003C\u002Fa>, and in this specific project, moderators demonstrated their understanding of unfriendly and unwelcoming behavior in comments.\n\nRegular registered users from our research list perceived the next lowest rate of unfriendly comments, and users who responded to our blog post about \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F04\u002F26\u002Fstack-overflow-isnt-very-welcoming-its-time-for-that-to-change\u002F\">making Stack Overflow more welcoming\u003C\u002Fa> found the lowest rates of unfriendly comments of all. How can we interpret this? We specifically invited users who may not consider themselves active participants in our community in order to gain outside perspective, but then these users saw the lowest rates of unwelcoming behavior.\n\nA possible explanation is that we are seeing a real effect of deep experience with our site; it appears the more invested an individual is here at Stack Overflow, the more sensitive they are to problematic behavior. What do these unfriendly comments look like? The following combine elements of real comments to show typical examples.\n\u003Cul>\n \t\u003Cli>\"\u003Cem>Why\u003C\u002Fem> do you want to do this? You have conflated at least three problems here.\"\u003C\u002Fli>\n \t\u003Cli>\"It will be very hard to help you with such a trivial bug. It could come from any line in your code, and we have to guess.\"\u003C\u002Fli>\n \t\u003Cli>\"How exactly is this going to solve my problem?!\"\u003C\u002Fli>\n \t\u003Cli>\"You don't understand how to use this site. Here nobody codes for you; read the docs and then show us.\"\u003C\u002Fli>\n \t\u003Cli>\"What are you \u003Cem>actually\u003C\u002Fem> trying to achieve? Please learn how to use a debugger.\"\u003C\u002Fli>\n\u003C\u002Ful>\nOur project showed that the more deeply an individual is connected to Stack Overflow (as an employee, or a moderator), the more they are likely to see problems in comments like these. \u003Cspan style=\"font-weight: 400;\">This effect is robust to comparing groups who were shown the same comments, who rated the same number of comments, and other analytical approaches.\u003C\u002Fspan>\n\u003Ch2>Individual differences\u003C\u002Fh2>\nWhat do the distributions of ratings for each individual look like?\n\n\u003Cimg class=\"aligncenter size-large wp-image-9532\" src=\"https:\u002F\u002Fstackoverflow.blog\u002Fwp-content\u002Fuploads\u002F2018\u002F11\u002Frating_boxplot-1-1200x508.png\" alt=\"\" width=\"1024\" height=\"433\">\n\nEach individual did not rate the same set of comments and worked for different lengths of time, so we expect variability in the results for each individual. Overall, the median percentage of perceived unwelcoming comments per individual was 3.5%, quite a bit lower than the \u003Ca href=\"https:\u002F\u002Fstackoverflow.blog\u002F2018\u002F07\u002F10\u002Fwelcome-wagon-classifying-comments-on-stack-overflow\u002F\">median percentage for employees\u003C\u002Fa> of 6.5%.\n\nTo understand how much agreement there is between raters, we can again look at \u003Ca href=\"https:\u002F\u002Fen.wikipedia.org\u002Fwiki\u002FKrippendorff%27s_alpha\">Krippendorff's alpha\u003C\u002Fa>, a measure that ranges from zero (nobody agrees) to one (perfect agreement). This measure accounts for the number of raters, so we can compare agreement among employees to the groups with more raters. What is Krippendorff's alpha, for comments that were rated by at least three people?\n\n\u003Cimg class=\"aligncenter size-large wp-image-9534\" src=\"https:\u002F\u002Fstackoverflow.blog\u002Fwp-content\u002Fuploads\u002F2018\u002F11\u002Fkripp_alpha-1-900x675.png\" alt=\"\" width=\"900\" height=\"675\">\n\nThese values for alpha are low compared to what social scientists would use to draw reliable conclusions based on the ratings; social scientists look for values close to 0.8 or more. Notice that Stack Overflow employees rated more comments as unwelcoming than other groups but agreed with each other about what is unwelcoming and abusive at higher rates at the same time. The rate of agreement among moderators and registered users was lowest (although still \u003Ca href=\"https:\u002F\u002Fstats.stackexchange.com\u002Fquestions\u002F33566\u002Fis-joel-spolskys-hunting-of-the-snark-post-valid-statistical-content-analysis\">much higher than for people unfamiliar with Stack Overflow\u003C\u002Fa>), and the rate of agreement for the users who volunteered to help make Stack Overflow more welcoming was a bit higher. Remember that these were the users who rated the \u003Cem>lowest\u003C\u002Fem> overall levels of unfriendliness; some spot-checking indicates these users identified only the clearest examples of problematic text.\n\nAnother factor that impacts interactions on Stack Overflow is \u003Ca href=\"https:\u002F\u002Fstackoverflow.com\u002Fhelp\u002Fwhats-reputation\">reputation\u003C\u002Fa>. Do we see any difference in how raters perceived unwelcoming behavior with their own reputation? This can help us understand if \"power users\" (distinguished from moderators) may be driving problems with site culture.\n\n\u003Cimg class=\"aligncenter size-large wp-image-9533\" src=\"https:\u002F\u002Fstackoverflow.blog\u002Fwp-content\u002Fuploads\u002F2018\u002F11\u002Freputation-1-1080x675.png\" alt=\"\" width=\"1024\" height=\"640\">\n\nThere is no clear evidence in this plot for a relationship, indicating that high-reputation users perceive unfriendly behavior at about the same rate as low-reputation users. We have seen effects similar to this before, for example, in our annual Developer Survey. When asked what the \u003Ca href=\"https:\u002F\u002Finsights.stackoverflow.com\u002Fsurvey\u002F2018#community-developers-perspectives-about-our-community\">worst or most annoying thing about Stack Overflow is\u003C\u002Fa>, developers of all experience levels and self-reported activity levels on Stack Overflow mentioned issues with harsh interactions and site culture.\n\nAll together, this begins to paint a complex and interesting picture of who understands unwelcoming behavior and in what ways. Moderators and high-reputation users are just as likely, or even more likely, to identify unwelcoming comments compared to new users. Stack Overflow employees identify more comments as problematic and agree with each other more about what is a problem compared to the other kinds of users in this project.\n\u003Ch2>Next steps\u003C\u002Fh2>\nSo where do we go from here? For starters, we as employees learned that we don't always perceive problems in the same way as other members of our community. We will keep this in mind as we move forward with plans to make Stack Overflow a better place for developers to learn and share knowledge.\n\nWe plan to use this dataset to investigate how comments are used on questions and answers, toward users of different experience levels, in different communities, and more. Look for more blog posts on these issues in upcoming months. We will continue to use the results from this project in product changes on Stack Overflow, as well as directly using appropriate subsets of this data in machine learning models. Also, in 2019 we will release this dataset (comment IDs, comment ratings, and anonymized\u002Frandomized rater ID) upon request so that other people in our community and beyond can explore this data for themselves.\n\nAll of this would not be possible without the investment of time and energy of the individuals who participated in this project, and we want to acknowledge each of you who volunteered to help us understand this aspect of our site better. Thank you, for your care and time. Community is central to our identity at Stack Overflow, and we are committed to making Stack Overflow a healthy, inclusive place for developers to learn and share knowledge.","html","2018-12-04T15:00:27.000Z",{"current":473},"welcome-wagon-community-and-comments-on-stack-overflow",[475,483,488,493,498],{"_createdAt":476,"_id":477,"_rev":478,"_type":479,"_updatedAt":476,"slug":480,"title":482},"2023-05-23T16:43:21Z","wp-tagcat-announcements","9HpbCsT2tq0xwozQfkc4ih","blogTag",{"current":481},"announcements","Announcements",{"_createdAt":476,"_id":484,"_rev":478,"_type":479,"_updatedAt":476,"slug":485,"title":487},"wp-tagcat-community",{"current":486},"community","Community",{"_createdAt":476,"_id":489,"_rev":478,"_type":479,"_updatedAt":476,"slug":490,"title":492},"wp-tagcat-company",{"current":491},"company","Company",{"_createdAt":476,"_id":494,"_rev":478,"_type":479,"_updatedAt":476,"slug":495,"title":497},"wp-tagcat-insights",{"current":496},"insights","Insights",{"_createdAt":476,"_id":499,"_rev":478,"_type":479,"_updatedAt":476,"slug":500,"title":502},"wp-tagcat-stackoverflow",{"current":501},"stackoverflow","Stackoverflow","Welcome Wagon: Community and Comments on Stack Overflow",[505,511,517,523],{"_id":506,"publishedAt":507,"slug":508,"sponsored":12,"title":510},"28e560af-f0aa-4d46-bd90-f435ad604aa7","2026-06-26T14:00:27.102Z",{"_type":10,"current":509},"paging-charity-how-can-engineering-leaders-avoid-becoming-bond-villains","Paging Charity! How can engineering leaders avoid becoming Bond villains?",{"_id":512,"publishedAt":513,"slug":514,"sponsored":12,"title":516},"4b22c2a3-3779-4966-93eb-5230391dbdce","2026-06-23T14:08:58.595Z",{"_type":10,"current":515},"your-ai-shipped-a-backend-that-boots-that-is-the-whole-problem","Your AI shipped a backend that boots. That is the whole problem.",{"_id":518,"publishedAt":519,"slug":520,"sponsored":12,"title":522},"5cf362e1-fe7b-45af-b69c-914731c6a052","2026-06-23T14:00:00.000Z",{"_type":10,"current":521},"the-2026-developer-survey-is-now-open-for-human-developers-only","The 2026 Developer Survey is now open (for human developers only)!",{"_id":524,"publishedAt":525,"slug":526,"sponsored":12,"title":528},"30b995f7-7cb9-4dd8-bf71-d0685940a32b","2026-06-19T14:00:00.000Z",{"_type":10,"current":527},"dispatches-from-o-reilly-from-capabilities-to-responsibilities","Dispatches from O'Reilly: From capabilities to responsibilities",{"data":530,"sourceMap":-1},{"count":531,"lastTimestamp":532},36,"2023-05-25T09:46:41Z"]