[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"sanity-sqUw5jV1-uiLkC4HWvYseKlOCwlBDv9tViUIBbFbIgI":3,"sanity-9cdDlZojTI250Bu1_sb03-ixSxFMqpIBD1DjGS6uvgA":322},{"data":4,"sourceMap":-1},{"latestPodcast":5,"latestReleases":14,"post":39,"recent":297},[6],{"_id":7,"publishedAt":8,"slug":9,"sponsored":12,"title":13},"4d0175f4-40a8-47eb-9bb3-a453b326aa7d","2026-07-03T07:40:00.000Z",{"_type":10,"current":11},"slug","the-good-the-bad-and-the-ai-apps",null,"The good, the bad, and the AI apps",[15,21,27,33],{"_id":16,"publishedAt":17,"slug":18,"title":20},"eb5b66eb-9410-4329-83bb-22bbff39402a","2026-04-28T13:00:00.000Z",{"_type":10,"current":19},"turn-scattered-knowledge-into-trusted-intelligence","Turning scattered knowledge into trusted intelligence: Stack Internal 2026.3",{"_id":22,"publishedAt":23,"slug":24,"title":26},"369c2401-b62e-4a37-8ff8-bf603023ecad","2026-03-02T15:03:00.988Z",{"_type":10,"current":25},"what-s-new-at-stack-overflow-march-2026","What’s new at Stack Overflow: March 2026",{"_id":28,"publishedAt":29,"slug":30,"title":32},"5e9053a4-07ea-447c-91ea-29e0b6228537","2026-02-02T15:00:00.000Z",{"_type":10,"current":31},"what-s-new-at-stack-overflow-february-2026","What’s new at Stack Overflow: February 2026",{"_id":34,"publishedAt":35,"slug":36,"title":38},"a1b538eb-a8a6-46d0-80a1-ac70ec9bb935","2026-01-05T10:00:00.000-05:00",{"_type":10,"current":37},"what-s-new-at-stack-overflow-january-2026","What’s new at Stack Overflow: January 2026",{"_createdAt":40,"_id":41,"_rev":42,"_system":43,"_type":46,"_updatedAt":47,"author":48,"body":65,"comments":263,"dateUrl":264,"excerpt":265,"image":266,"product":12,"publishedAt":270,"slug":271,"sponsored":12,"tags":273,"title":296,"visible":263},"2024-09-24T19:24:17Z","484c1547-c0ce-40f6-a02b-8e59ec0d4d29","OuLxq6F1ZwNJHk6WTQWyiK",{"base":44},{"id":41,"rev":45},"8y8N0FZhvd5mVuRpOn2O3x","blogPost","2026-05-11T17:06:03Z",[49],{"_createdAt":50,"_id":51,"_rev":52,"_type":53,"_updatedAt":50,"avatar":54,"bio":59,"employee":60,"name":61,"role":62,"slug":63},"2024-04-04T14:34:48Z","12169ed9-78d5-4f82-adb4-e956348db389","l85YFb2NNiT2L5SEvOgP0n","blogAuthor",{"_type":55,"asset":56},"image",{"_ref":57,"_type":58},"image-7c75ba835850e53201ea979015e8b8f449e126da-400x400-png","reference","I’m currently the Director of AI at Rebuy, a personalized search and recommendations platform for D2C e-commerce brands. Prior to Rebuy, I was a Research Scientist at Alegion. Additionally, I worked for Salesforce Commerce Cloud for two years. ","none","Cameron R. Wolfe, PhD","Director of AI at Rebuy",{"_type":10,"current":64},"cameron-r-wolfe-phd",[66,89,97,114,130,137,145,152,164,171,183,190,202,209,221,228,240,247,259],{"_key":67,"_type":68,"children":69,"markDefs":84,"style":88},"7133285ca744","block",[70,75,80],{"_key":71,"_type":72,"marks":73,"text":74},"84daaa34f6450","span",[],"The big picture: Large language models are based upon a deep neural network architecture called a ",{"_key":76,"_type":72,"marks":77,"text":79},"84daaa34f6451",[78],"5807f91382c9","decoder-only transformer",{"_key":81,"_type":72,"marks":82,"text":83},"84daaa34f6452",[],". Within each layer of this model, we have two key components:",[85],{"_key":78,"_type":86,"href":87,"reference":12},"link","https:\u002F\u002Fstackoverflow.blog\u002F2024\u002F08\u002F22\u002Fllms-evolve-quickly-their-underlying-architecture-not-so-much\u002F","normal",{"_key":90,"_type":68,"children":91,"markDefs":96,"style":88},"1ebf0f48fa02",[92],{"_key":93,"_type":72,"marks":94,"text":95},"058231bfee270",[],"",[],{"_key":98,"_type":68,"children":99,"markDefs":113,"style":88},"7857932d1cac",[100,104,109],{"_key":101,"_type":72,"marks":102,"text":103},"61a5d26e89ed0",[],"1. ",{"_key":105,"_type":72,"marks":106,"text":108},"fe6853d032c5",[107],"strong","Masked self-attention ",{"_key":110,"_type":72,"marks":111,"text":112},"98de591a4c9a",[],"learns relationships between tokens\u002Fwords.",[],{"_key":115,"_type":68,"children":116,"markDefs":129,"style":88},"9b89053754ac",[117,121,125],{"_key":118,"_type":72,"marks":119,"text":120},"6a84eb5ec9510",[],"2. ",{"_key":122,"_type":72,"marks":123,"text":124},"a5532a02bb95",[107],"Feed-forward transformation",{"_key":126,"_type":72,"marks":127,"text":128},"2d38a2504ed0",[]," individually transforms the representation of each word.",[],{"_key":131,"_type":68,"children":132,"markDefs":136,"style":88},"ed369d140bcb",[133],{"_key":134,"_type":72,"marks":135,"text":95},"48520ca34fb70",[],[],{"_key":138,"_type":68,"children":139,"markDefs":144,"style":88},"a08987685944",[140],{"_key":141,"_type":72,"marks":142,"text":143},"2d49ab49f9dd0",[],"These components are complementary—attention looks across the sequence, while feed-forward transformations consider each token individually. When combined together, they allow us to learn complex patterns from text that power the AI applications that are so popular today.",[],{"_key":146,"_type":68,"children":147,"markDefs":151,"style":88},"660ac405ece3",[148],{"_key":149,"_type":72,"marks":150,"text":95},"e8b843b45a750",[],[],{"_key":153,"_type":68,"children":154,"markDefs":163,"style":88},"87780f37a4ae",[155,159],{"_key":156,"_type":72,"marks":157,"text":158},"36d5863d8b5b0",[107],"TL;DR:",{"_key":160,"_type":72,"marks":161,"text":162},"753db275e0c3",[]," The input to an attention model is a list of token\u002Fword vectors, which can be stacked together to form a matrix. Causal self-attention operates by computing an attention\u002Fimportance score between each pair of tokens\u002Fwords in a sequence. Then, the output of self-attention is a weighted combination of all words in the sequence, where the weight is given by the attention score. We can break the process of masked self-attention into a sequence of five steps.",[],{"_key":165,"_type":68,"children":166,"markDefs":170,"style":88},"6cadb2830b21",[167],{"_key":168,"_type":72,"marks":169,"text":95},"ecb56ae50cd10",[],[],{"_key":172,"_type":68,"children":173,"markDefs":182,"style":88},"c67000eea455",[174,178],{"_key":175,"_type":72,"marks":176,"text":177},"2ca1cab3bffa0",[107],"(1) Linear projections: ",{"_key":179,"_type":72,"marks":180,"text":181},"11a8e84465a7",[],"The first step is to perform three separate linear projections, called the query, key, and value projections. Practically, these projections take our sequence of token vectors as input and produce three transformed sequences of token vectors as output.",[],{"_key":184,"_type":68,"children":185,"markDefs":189,"style":88},"1640d21782e2",[186],{"_key":187,"_type":72,"marks":188,"text":95},"d3d993dd2ee60",[],[],{"_key":191,"_type":68,"children":192,"markDefs":201,"style":88},"faf0cf6235de",[193,197],{"_key":194,"_type":72,"marks":195,"text":196},"8ab4ab8adff30",[107],"(2) Attention scores: ",{"_key":198,"_type":72,"marks":199,"text":200},"6bcaf303d83b",[],"To compute attention scores, we use the query and key vectors produced by the linear projections described above. The attention score between the i-th token and the j-th token in the sequence is given by the dot product of the i-th query vector and the j-th key vector. To compute all of these pairwise scores efficiently, we can stack the query\u002Fkey vectors into matrices and take the matrix product of the query matrix with the transposed key matrix. The output is a TxT attention matrix, where T is the length of the input sequence (in tokens). To improve training stability, we also divide the values of the attention matrix by the square root of the size of the token vectors (i.e., scaled dot product attention).",[],{"_key":203,"_type":68,"children":204,"markDefs":208,"style":88},"31e718d0f187",[205],{"_key":206,"_type":72,"marks":207,"text":95},"70ee9aac504f0",[],[],{"_key":210,"_type":68,"children":211,"markDefs":220,"style":88},"1d513bdfebd4",[212,216],{"_key":213,"_type":72,"marks":214,"text":215},"c63674e936fe0",[107],"(3) Forming a probability distribution: ",{"_key":217,"_type":72,"marks":218,"text":219},"2f83953455ce",[],"From here, we can turn the attention scores for each token into a probability distribution by performing a softmax operation across each token’s attention scores for the sequence. In practice, this is done via a softmax operation across each row of the attention matrix. After this, each row of the attention matrix becomes a probability distribution that represents the (normalized) attention scores for a single token across the sequence (i.e., the i-th row contains the i-th token’s attention scores).",[],{"_key":222,"_type":68,"children":223,"markDefs":227,"style":88},"269296639ac3",[224],{"_key":225,"_type":72,"marks":226,"text":95},"c66f687dc0cd0",[],[],{"_key":229,"_type":68,"children":230,"markDefs":239,"style":88},"24eec03f7ede",[231,235],{"_key":232,"_type":72,"marks":233,"text":234},"c44b9057bc520",[107],"(4) Masking operation: ",{"_key":236,"_type":72,"marks":237,"text":238},"c43267736a74",[],"In vanilla self-attention, each token is allowed to compute attention scores for all tokens in the sequence. In masked self-attention, however, we mask attention scores for any token that follows a given token in the sequence. We can implement this by simply masking the attention matrix prior to performing the softmax (i.e., fill entries for any invalid attention scores with a value of negative infinity), such that the probability of any future token in the sequence becomes zero. For example, the i-th token in the sequence would have an attention scores of 0 for tokens i + 1, i + 2, and so on. Practically, masked self-attention prevents us from looking forward in the sequence when computing a token’s representation.",[],{"_key":241,"_type":68,"children":242,"markDefs":246,"style":88},"d8ff7f9d52b8",[243],{"_key":244,"_type":72,"marks":245,"text":95},"680d7b5b79390",[],[],{"_key":248,"_type":68,"children":249,"markDefs":258,"style":88},"c484bbffa104",[250,254],{"_key":251,"_type":72,"marks":252,"text":253},"faa5c3f428ad0",[107],"(5) Computing the output: ",{"_key":255,"_type":72,"marks":256,"text":257},"4e83b534d5eb",[],"From here, we can compute the output of masked self-attention by taking the matrix product of the attention matrix and a matrix of value vectors. This operation computes the output for the i-th token by taking a weighted combination of all value vectors, where the weights are given by token i’s attention scores.",[],{"_key":260,"_type":55,"asset":261,"markDefs":12},"d4263045a469",{"_ref":262,"_type":58},"image-25ebbba9d2ce12efc8c3da181942367f05c795be-2386x1338-jpg",true,"2024\u002F09\u002F26","Masked self-attention is the key building block that allows LLMs to learn rich relationships and patterns between the words of a sentence. Let’s build it together from scratch.",{"_type":55,"asset":267,"attribution":269},{"_ref":268,"_type":58},"image-e278dfb29b78d1ba33dc27e79d5906eb53acf972-12000x6300-jpg","Alexandra Francis","2024-09-26T16:00:00.000Z",{"_type":10,"current":272},"masked-self-attention-how-llms-learn-relationships-between-tokens",[274,281,285,289],{"_createdAt":275,"_id":276,"_rev":277,"_type":278,"_updatedAt":275,"slug":279,"title":280},"2023-05-23T16:43:21Z","wp-tagcat-se-tech","9HpbCsT2tq0xwozQfkc4ih","blogTag",{"current":280},"se-tech",{"_createdAt":275,"_id":282,"_rev":277,"_type":278,"_updatedAt":275,"slug":283,"title":284},"wp-tagcat-se-stackoverflow",{"current":284},"se-stackoverflow",{"_createdAt":275,"_id":286,"_rev":277,"_type":278,"_updatedAt":275,"slug":287,"title":288},"wp-tagcat-llm",{"current":288},"llm",{"_createdAt":275,"_id":290,"_rev":291,"_type":278,"_updatedAt":292,"slug":293,"title":295},"wp-tagcat-ai","fpDTFQqIDjNJIbHDKPBGpV","2025-01-30T16:19:01Z",{"current":294},"ai","AI","Masked self-attention: How LLMs learn relationships between tokens",[298,304,310,316],{"_id":299,"publishedAt":300,"slug":301,"sponsored":12,"title":303},"28e560af-f0aa-4d46-bd90-f435ad604aa7","2026-06-26T14:00:27.102Z",{"_type":10,"current":302},"paging-charity-how-can-engineering-leaders-avoid-becoming-bond-villains","Paging Charity! How can engineering leaders avoid becoming Bond villains?",{"_id":305,"publishedAt":306,"slug":307,"sponsored":12,"title":309},"4b22c2a3-3779-4966-93eb-5230391dbdce","2026-06-23T14:08:58.595Z",{"_type":10,"current":308},"your-ai-shipped-a-backend-that-boots-that-is-the-whole-problem","Your AI shipped a backend that boots. That is the whole problem.",{"_id":311,"publishedAt":312,"slug":313,"sponsored":12,"title":315},"5cf362e1-fe7b-45af-b69c-914731c6a052","2026-06-23T14:00:00.000Z",{"_type":10,"current":314},"the-2026-developer-survey-is-now-open-for-human-developers-only","The 2026 Developer Survey is now open (for human developers only)!",{"_id":317,"publishedAt":318,"slug":319,"sponsored":12,"title":321},"30b995f7-7cb9-4dd8-bf71-d0685940a32b","2026-06-19T14:00:00.000Z",{"_type":10,"current":320},"dispatches-from-o-reilly-from-capabilities-to-responsibilities","Dispatches from O'Reilly: From capabilities to responsibilities",{"data":323,"sourceMap":-1},{"count":324,"lastTimestamp":325},1,"2024-09-30T15:59:46Z"]