{"id":8227,"date":"2023-08-16T14:10:56","date_gmt":"2023-08-16T05:10:56","guid":{"rendered":"https:\/\/agirobots.com\/?p=8227"},"modified":"2023-10-24T10:30:29","modified_gmt":"2023-10-24T01:30:29","slug":"rwkv","status":"publish","type":"post","link":"https:\/\/developers.agirobots.com\/jp\/rwkv\/","title":{"rendered":"RWKV\u306b\u3064\u3044\u3066\u89e3\u8aac"},"content":{"rendered":"\n

\u672c\u8a18\u4e8b\u3067\u306f\u3001RWKV\u3068\u3088\u3070\u308c\u308b\u30e2\u30c7\u30eb\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u306b\u3064\u3044\u3066\u8a73\u3057\u304f\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n

\u306f\u3058\u3081\u306b<\/h2>\n\n\n\n

\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u5206\u91ce\u306b\u304a\u3044\u3066\u3001Transformer[1]\u306e\u767b\u5834\u4ee5\u524d\u306b\u4e00\u822c\u7684\u306b\u4f7f\u7528\u3055\u308c\u3066\u304d\u305fRNN[2]\u306f\u3044\u304f\u3064\u304b\u306e\u8ab2\u984c\u3092\u62b1\u3048\u3066\u304a\u308a\u3001\u305d\u306e\u8ab2\u984c\u3092\u514b\u670d\u3059\u308b\u65b0\u305f\u306a\u624b\u6cd5\u3068\u3057\u3066\u3001RNN\u3068\u306f\u5b8c\u5168\u306b\u7570\u306a\u308b\u30a2\u30d7\u30ed\u30fc\u30c1\u3092\u53d6\u308bTransformer\u304c\u767b\u5834\u3057\u307e\u3057\u305f\u3002\u3057\u304b\u3057\u3001Transformer\u306b\u3082\u89e3\u6c7a\u3059\u3079\u304d\u554f\u984c\u304c\u5b58\u5728\u3057\u3066\u3044\u307e\u3059\u3002\u305d\u3053\u3067\u3001\u3053\u308c\u3089\u306e\u30a2\u30d7\u30ed\u30fc\u30c1\u3092\u7d50\u3073\u3064\u3051\u3066\u9032\u5316\u3055\u305b\u3066\u3044\u304f\u5fc5\u8981\u304c\u51fa\u3066\u304d\u307e\u3057\u305f\u3002<\/p>\n\n\n\n

\u307e\u305a\u3001RNN\u306e\u5229\u70b9\u3068\u6b20\u70b9\u3092\u898b\u3066\u307f\u307e\u3057\u3087\u3046\u3002RNN\u306f\u3001\u6587\u7ae0\u306e\u9577\u3055\u306b\u307b\u3068\u3093\u3069\u5236\u7d04\u304c\u306a\u304f\u3001\u8a08\u7b97\u30b3\u30b9\u30c8\u3082\u6bd4\u8f03\u7684\u5c0f\u3055\u3044\u3068\u3044\u3046\u5229\u70b9\u304c\u3042\u308a\u307e\u3059\u3002\u3057\u304b\u3057\u3001\u4ee5\u524d\u306e\u5165\u529b\u3092\u6b63\u78ba\u306b\u8a18\u61b6\u3059\u308b\u3053\u3068\u304c\u96e3\u3057\u304f\uff08\u9577\u671f\u4f9d\u5b58\u6027\u3092\u6349\u3048\u3089\u308c\u306a\u3044\uff09\u3001\u5b66\u7fd2\u3092\u9ad8\u901f\u5316\u3059\u308b\u3053\u3068\u3082\u96e3\u3057\u3044\uff08\u5b66\u7fd2\u4e26\u5217\u5316\u304c\u56f0\u96e3\uff09\u3068\u3044\u3046\u6b20\u70b9\u3082\u5b58\u5728\u3057\u307e\u3059\u3002<\/p>\n\n\n\n

\u4e00\u65b9\u3001Transformer\u306f\u9577\u671f\u4f9d\u5b58\u6027\u3092\u6349\u3048\u308b\u3053\u3068\u304c\u53ef\u80fd\u3067\u6027\u80fd\u3082\u9ad8\u3044\u3068\u3044\u3046\u5229\u70b9\u304c\u3042\u308a\u307e\u3059\u3002\u307e\u305f\u3001\u5b66\u7fd2\u3092\u52b9\u7387\u7684\u306b\u9032\u3081\u308b\u3053\u3068\u304c\u3067\u304d\u308b\uff08\u5b66\u7fd2\u4e26\u5217\u5316\u304c\u53ef\u80fd\uff09\u3068\u3044\u3046\u9577\u6240\u3082\u3042\u308a\u307e\u3059\u3002\u3057\u304b\u3057\u306a\u304c\u3089\u3001\u8a08\u7b97\u30b3\u30b9\u30c8\u3084\u5fc5\u8981\u306a\u30e1\u30e2\u30ea\u30b5\u30a4\u30ba\u304c\u7cfb\u5217\u9577\u306b\u5fdc\u3058\u3066\u6025\u6fc0\u306b\u5897\u52a0\u3059\u308b\u3068\u3044\u3046\u554f\u984c\u3082\u3042\u308a\u3001RNN\u3068\u540c\u69d8\u306b\u7406\u8ad6\u4e0a\u306f\u6271\u3048\u308b\u7cfb\u5217\u9577\u306f\u7121\u9650\u3067\u3059\u304c\u3001\u5b9f\u904b\u7528\u4e0a\u306f\u30cf\u30fc\u30c9\u30a6\u30a7\u30a2\u306e\u5236\u7d04\u3092\u5f37\u304f\u53d7\u3051\u308b\u3053\u3068\u306b\u306a\u308a\u3001RNN\u306b\u6bd4\u3079\u3066\u9577\u3044\u7cfb\u5217\u3092\u6271\u3044\u306b\u304f\u3044\u3068\u3044\u3046\u6b20\u70b9\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n

\u3053\u3046\u3057\u3066\u307f\u308b\u3068\u3001\u305d\u308c\u305e\u308c\u306e\u624b\u6cd5\u306e\u5229\u70b9\u3068\u6b20\u70b9\u306f\u6b63\u53cd\u5bfe\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002\u3067\u3059\u304b\u3089\u3001\u4e21\u8005\u306e\u5229\u70b9\u3092\u7d50\u5408\u3057\u305f\u30e2\u30c7\u30eb\u3001\u3064\u307e\u308a\u3001RNN\u306e\u3088\u3046\u306b\u9577\u3044\u6587\u7ae0\u3092\u6271\u3046\u3053\u3068\u304c\u3067\u304d\u3001\u8a08\u7b97\u30b3\u30b9\u30c8\u304c\u5c0f\u3055\u304f\u3001Transformer\u306e\u3088\u3046\u306b\u9577\u671f\u306e\u4f9d\u5b58\u95a2\u4fc2\u3082\u6349\u3048\u3089\u308c\u3001\u3055\u3089\u306b\u5b66\u7fd2\u3092\u4e26\u5217\u5316\u3057\u3066\u9ad8\u901f\u5316\u3067\u304d\u308b\u30e2\u30c7\u30eb\u306e\u5b9f\u73fe\u3092\u76ee\u6307\u3059\u6d41\u308c\u306f\u5fc5\u7136\u3068\u8a00\u3048\u308b\u3067\u3057\u3087\u3046\u3002<\/p>\n\n\n\n

\u3053\u306e\u3088\u3046\u306a\u7406\u60f3\u7684\u306a\u30e2\u30c7\u30eb\u306e\u5b9f\u73fe\u306b\u5411\u3051\u305f\u7b2c\u4e00\u6b69\u304c\u3001\u4eca\u56de\u7d39\u4ecb\u3059\u308bRWKV\uff08Receptance Weighted Key Value\uff09[3]\u3067\u3059\u3002<\/p>\n\n\n\n

\u672c\u8a18\u4e8b\u3067\u306f\u3001RWKV\u306e\u4ed5\u7d44\u307f\u306b\u3064\u3044\u3066\u5206\u304b\u308a\u3084\u3059\u304f\u89e3\u8aac\u3057\u3066\u3044\u304d\u307e\u3059\u3002<\/p>\n\n\n\n

RWKV\u767b\u5834\u307e\u3067\u306e\u6d41\u308c<\/h2>\n\n\n\n

Transformer\u306e\u767b\u5834<\/h3>\n\n\n\n

RWKV\u306e\u767b\u5834\u307e\u3067\u306e\u7d4c\u7def\u306b\u3064\u3044\u3066\u3001\u5148\u307b\u3069\u3088\u308a\u3082\u8a73\u3057\u304f\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n

Transformer\u304c\u767b\u5834\u3059\u308b\u4ee5\u524d\u3001\u6df1\u5c64\u5b66\u7fd2\u306f\u6a5f\u68b0\u5b66\u7fd2\u5206\u91ce\u3067\u5927\u304d\u306a\u6210\u529f\u3092\u53ce\u3081\u3066\u3044\u307e\u3057\u305f\u3002\u7279\u306b\u6ce8\u76ee\u3059\u3079\u304d\u306f\u3001\u753b\u50cf\u51e6\u7406\u3067\u6210\u679c\u3092\u4e0a\u3052\u305fCNN[4]\u3068\u3001\u7cfb\u5217\u51e6\u7406\u3067\u6210\u679c\u3092\u4e0a\u3052\u305fRNN\u3067\u3059\u3002\u3057\u304b\u3057\u306a\u304c\u3089\u3001\u3053\u308c\u3089\u306e\u624b\u6cd5\u306b\u3082\u8ab2\u984c\u304c\u5b58\u5728\u3057\u3066\u3044\u307e\u3057\u305f\u3002<\/p>\n\n\n\n

\u5177\u4f53\u7684\u306b\u3001RNN\u306b\u306f\u9577\u671f\u4f9d\u5b58\u6027\u3092\u6349\u3048\u308b\u3053\u3068\u304c\u56f0\u96e3\u3001\u305d\u3057\u3066\u5b66\u7fd2\u306e\u9ad8\u901f\u5316\u304c\u56f0\u96e3\u3068\u3044\u30462\u3064\u306e\u8ab2\u984c\u304c\u3042\u308a\u307e\u3057\u305f\u3002\u9577\u671f\u4f9d\u5b58\u6027\u3092\u6349\u3048\u308b\u306e\u304c\u96e3\u3057\u3044\u306e\u306f\u3001\u5b66\u7fd2\u6642\u306b\u904e\u53bb\u306e\u60c5\u5831\u3092\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3059\u308b\u969b\u306b\u3001\u9014\u4e2d\u3067\u52fe\u914d\u304c\u6d88\u5931\u3057\u3066\u3057\u307e\u3044\u3001\u9577\u671f\u7684\u306a\u7279\u5fb4\u3092\u3046\u307e\u304f\u5b66\u7fd2\u3067\u304d\u306a\u3044\u305f\u3081\u3067\u3059\u3002\u307e\u305f\u3001\u5b66\u7fd2\u306e\u9ad8\u901f\u5316\u304c\u96e3\u3057\u3044\u306e\u306f\u3001\u5165\u529b\u7cfb\u5217\u3092\u6642\u523b\u3054\u3068\u306b\u9010\u6b21\u7684\u306b\u51e6\u7406\u3059\u308b\u5fc5\u8981\u304c\u3042\u308b\u305f\u3081\u3067\u3059\u3002RNN\u306f\u5185\u90e8\u306b\u518d\u5e30\u69cb\u9020\u3092\u6301\u3064\u305f\u3081\u3001\u904e\u53bb\u306e\u60c5\u5831\u3092\u4fdd\u6301\u3057\u306a\u304c\u3089\u51e6\u7406\u3092\u9032\u3081\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002\u9006\u306b\u8a00\u3048\u3070\u3001\u5404\u6642\u523b\u3067\u306e\u8a08\u7b97\u30b3\u30b9\u30c8\u306f\u5c0f\u3055\u3044\u3068\u3044\u3046\u5229\u70b9\u3082\u3042\u308a\u307e\u3059\u3002RNN\u306e\u6b20\u70b9\u3092\u88dc\u3046\u624b\u6bb5\u3068\u3057\u3066CNN\u3092\u5c0e\u5165\u3059\u308b\u3068\u3044\u3046\u3082\u306e\u304c\u3042\u308a\u307e\u3059\u304c\u3001\u305d\u308c\u306b\u3082\u9650\u754c\u304c\u3042\u308a\u307e\u3057\u305f\u3002CNN\u3067\u7cfb\u5217\u51e6\u7406\u3092\u884c\u3046\u969b\u306b\u306f\u30011\u6b21\u5143\u7573\u307f\u8fbc\u307f\u30d5\u30a3\u30eb\u30bf\u3092\u7528\u3044\u307e\u3059\u304c\u3001\u7573\u307f\u8fbc\u307f\u30d5\u30a3\u30eb\u30bf\u306f\u8003\u616e\u53ef\u80fd\u306a\u6642\u9593\u7684\u306a\u9818\u57df\u306b\u660e\u78ba\u306a\u7dda\u5f15\u304d\u304c\u3042\u308b\u305f\u3081\u3001\u9577\u671f\u7684\u306a\u7279\u5fb4\u3092\u6349\u3048\u308b\u306b\u306f\u591a\u5c64\u306b\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002\u305f\u3060\u3057\u3001\u8003\u616e\u53ef\u80fd\u306a\u9818\u57df\u3092\u7121\u9650\u306b\u3059\u308b\u306b\u306f\u5c64\u6570\u3082\u7121\u9650\u306b\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u73fe\u5b9f\u7684\u306a\u4ee3\u66ff\u624b\u6bb5\u306b\u306f\u306a\u308a\u307e\u305b\u3093\u3067\u3057\u305f\u3002<\/p>\n\n\n\n

\u305d\u3053\u3067\u3001\u65b0\u3057\u3044\u30e2\u30c7\u30eb\u3067\u3042\u308bTransformer\u304c\u751f\u307e\u308c\u305f\u306e\u3067\u3059\u3002Transformer\u306f\u3001\u7cfb\u5217\u5168\u4f53\u3092\u4e00\u62ec\u3057\u3066\u5165\u529b\u3068\u3057\u3066\u53d7\u3051\u53d6\u308a\u3001\u5168\u30c8\u30fc\u30af\u30f3\u9593\u306e\u7d44\u307f\u5408\u308f\u305b\u3092\u6bce\u56de\u8a08\u7b97\u3059\u308b\u3053\u3068\u3067\u9577\u671f\u4f9d\u5b58\u6027\u3092\u6349\u3048\u308b\u3053\u3068\u304c\u53ef\u80fd\u306b\u306a\u308a\u307e\u3057\u305f\u3002\u5185\u90e8\u306b\u518d\u5e30\u6027\u3092\u6301\u305f\u306a\u3044\u305f\u3081\u3001\u4e26\u5217\u51e6\u7406\u3082\u5b9f\u73fe\u3067\u304d\u308b\u3088\u3046\u306b\u306a\u3063\u305f\u306e\u3067\u3059\u3002<\/p>\n\n\n

\n
\"\"<\/figure><\/div>\n\n\n

Transformer\u306e\u767b\u5834\u306b\u3064\u3044\u3066\u306f\u4ee5\u4e0b\u306e\u8a18\u4e8b\u3067\u8a73\u3057\u304f\u8aac\u660e\u3057\u3066\u3044\u307e\u3059\u306e\u3067\u3001\u8208\u5473\u304c\u3042\u308a\u307e\u3057\u305f\u3089\u53c2\u8003\u306b\u3057\u3066\u304f\u3060\u3055\u3044\u3002<\/p>\n\n\n\n

\n
\"\"<\/dt>

Transformer\u306b\u3064\u3044\u3066\u672c\u8cea\u3092\u5206\u304b\u308a\u3084\u3059\u304f\u89e3\u8aac\uff01<\/p>

\u7686\u3055\u3093\u3053\u3093\u306b\u3061\u306f\uff01 \u53bb\u5e74\uff082022\u5e7411\u6708\uff09\u306b\u767a\u8868\u3055\u308c\u305fChatGPT\u306e\u8a71\u984c\u304c\u5c3d\u304d\u306a\u3044\u4eca\u65e5\u3053\u306e\u9803\u3001ChatGPT\u3067\u4f7f\u308f\u308c\u3066\u3044\u308b\u91cd\u8981\u306a\u6280\u8853\u306e\uff11\u3064\u3067\u3042\u308bTransformer\u306b\u3064\u3044\u3066\u3001\u8208\u5473\u3092\u6301\u3063\u305f\u65b9\u306f\u6ca2\u5c71 ... <\/p><\/div><\/dd><\/dl><\/div><\/a><\/div>\n<\/div><\/figure>\n\n\n\n

Transformer\u306e\u4e07\u80fd\u6027\u3068\u8ab2\u984c<\/h3>\n\n\n\n

Transformer\u306f\u5143\u3005\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u305f\u3081\u306b\u767b\u5834\u3057\u307e\u3057\u305f\u304c\u3001\u5f8c\u306b\u753b\u50cf\u8a8d\u8b58\u3067\u3082\u512a\u308c\u305f\u7d50\u679c\u3092\u793a\u3057\u307e\u3057\u305f\u3002\u591a\u304f\u306e\u7814\u7a76\u8005\u304cTransformer\u306e\u3053\u306e\u3088\u3046\u306a\u4e07\u80fd\u6027\u306b\u6ce8\u76ee\u3057\u3066\u3044\u307e\u3059\u304c\u3001\u306a\u305c\u305d\u3093\u306a\u306b\u512a\u308c\u3066\u3044\u308b\u306e\u3067\u3057\u3087\u3046\u304b\u3002<\/p>\n\n\n\n

\u7b54\u3048\u306f\u300c\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9[5]\u306e\u4f4e\u3055\u300d\u306b\u3042\u308a\u307e\u3059\u3002\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u3068\u306f\u3001\u30e2\u30c7\u30eb\u306e\u6c4e\u7528\u6027\u3092\u8aac\u660e\u3059\u308b\u969b\u306e\u8003\u3048\u65b9\u306e\u4e00\u3064\u3067\u3059\u3002\u4f8b\u3048\u3070\u3001\u753b\u50cf\u51e6\u7406\u306b\u306f\u7a7a\u9593\u30d5\u30a3\u30eb\u30bf\u3092\u6301\u3064\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u304c\u3001\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306b\u306f\u518d\u5e30\u69cb\u9020\u3092\u6301\u3064\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u304c\u9069\u3057\u3066\u3044\u308b\u3068\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u3053\u308c\u3089\u306f\u7279\u5b9a\u306e\u30bf\u30b9\u30af\u306b\u7279\u5316\u3057\u305f\u9ad8\u6027\u80fd\u306a\u30e2\u30c7\u30eb\u3067\u3001\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u304c\u9ad8\u3044\u3067\u3059\u3002\u9ad8\u3044\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u3092\u6301\u3064\u30e2\u30c7\u30eb\u306f\u3001\u3082\u3068\u3082\u3068\u60f3\u5b9a\u3055\u308c\u3066\u3044\u306a\u3044\u9818\u57df\u3067\u9ad8\u6027\u80fd\u3092\u5b9f\u73fe\u3059\u308b\u3053\u3068\u306f\u96e3\u3057\u3044\u3067\u3059\u3002<\/p>\n\n\n\n

\u305d\u308c\u306b\u5bfe\u3057\u3066\u3001Transformer\u306f\u3082\u3068\u3082\u3068\u60f3\u5b9a\u3055\u308c\u3066\u3044\u306a\u304b\u3063\u305f\u753b\u50cf\u51e6\u7406\u3084\u5f37\u5316\u5b66\u7fd2\u306a\u3069\u306e\u9818\u57df\u3067\u3082\u9ad8\u3044\u6027\u80fd\u3092\u5b9f\u73fe\u3057\u3066\u3044\u307e\u3059\u3002\u3053\u306e\u9069\u7528\u7bc4\u56f2\u306e\u5e83\u3055\u306f\u3001\u6c4e\u7528\u6027\u306e\u9ad8\u3044\u30e2\u30c7\u30eb\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u3092\u6301\u3063\u3066\u3044\u308b\u3053\u3068\u3092\u610f\u5473\u3057\u3001\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u304c\u4f4e\u3044\u3068\u8a00\u3048\u307e\u3059\u3002<\/p>\n\n\n\n

\u3053\u306e\u3001Transformer\u306e\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u306e\u4f4e\u3055\u304c\u4e07\u80fd\u3055\u306e\u79d8\u5bc6\u3067\u3059\u3002<\/p>\n\n\n\n

\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u306b\u3064\u3044\u3066\u306f\u4ee5\u4e0b\u306e\u8a18\u4e8b\u3067\u8a73\u3057\u304f\u8aac\u660e\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

\n
\"\"<\/dt>

\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u3068AI\u30e2\u30c7\u30eb\u306e\u672a\u6765\uff1a\u6c4e\u7528AI\u306e\u8a2d\u8a08\u306b\u304a\u3051\u308b\u65b0\u305f\u306a\u8996\u70b9<\/p>

\u6c4e\u7528\u4eba\u5de5\u77e5\u80fd\u306e\u5b9f\u73fe\u304c\u76ee\u524d\u306b\u8feb\u3063\u3066\u3044\u308b\u3088\u3046\u306b\u611f\u3058\u308b\u4eca\u65e5\u3053\u306e\u9803\u3001\u4eca\u5f8c\u306e\u6c4e\u7528\u4eba\u5de5\u77e5\u80fd\u7814\u7a76\u306f\u3069\u3046\u9032\u3080\u306e\u304b\u6c17\u306b\u306a\u308b\u3053\u3068\u304c\u591a\u304f\u306a\u308a\u307e\u3057\u305f\u3002 \u6c4e\u7528\u4eba\u5de5\u77e5\u80fd\u3092\u5b9f\u73fe\u3059\u308b\u306b\u306f\u3001\u4eba\u5de5\u77e5\u80fd\u306e\u7814\u7a76\u3060\u3051\u3067\u306a\u304f\u3001\u8133\u306e\u7814\u7a76\u3084\u4eba\u9593\u306e\u7814 ... <\/p><\/div><\/dd><\/dl><\/div><\/a><\/div>\n<\/div><\/figure>\n\n\n\n

Transformer\u306e\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u306e\u4f4e\u3055\u3092\u6d3b\u7528\u3057\u3066\u3001\u5165\u529b\u30e2\u30c0\u30ea\u30c6\u30a3\u306e\u5236\u7d04\u306e\u5c11\u306a\u3044\u3088\u308a\u6c4e\u7528\u7684\u306a\u30e2\u30c7\u30eb\u3068\u3057\u3066Perceiver[6]\u304c\u63d0\u6848\u3055\u308c\u3066\u3044\u307e\u3059\u3002Perceiver\u306b\u3064\u3044\u3066\u306f\u4ee5\u4e0b\u306e\u8a18\u4e8b\u3067\u8a73\u3057\u304f\u8aac\u660e\u3057\u3066\u3044\u307e\u3059\u306e\u3067\u3001\u8208\u5473\u304c\u3042\u308a\u307e\u3057\u305f\u3089\u662f\u975e\u8aad\u3093\u3067\u307f\u3066\u304f\u3060\u3055\u3044\u3002<\/p>\n\n\n\n

\n
\"\"<\/dt>

Perceiver\u30fbPerceiver IO\u306b\u3064\u3044\u3066\u89e3\u8aac\uff01<\/p>

\u306f\u3058\u3081\u306b \u7686\u3055\u3093\u306f\u3001\u65e5\u5e38\u751f\u6d3b\u306e\u4e2d\u3067\u3001\u8996\u899a\u3084\u8074\u899a\u3001\u89e6\u899a\u306a\u3069\u3001\u3055\u307e\u3056\u307e\u306a\u611f\u899a\u3092\u540c\u6642\u306b\u4f7f\u3063\u3066\u60c5\u5831\u3092\u51e6\u7406\u3057\u3066\u3044\u307e\u3059\u3002\u3053\u308c\u306f\u3001\u4eba\u9593\u304c\u8907\u6570\u306e\u611f\u899a\u60c5\u5831\u3092\u7d71\u5408\u7684\u306b\u77e5\u7684\u51e6\u7406\u3057\u3066\u3044\u308b\u3001\u3044\u308f\u3086\u308b\u300c\u30de\u30eb\u30c1\u30e2\u30fc\u30c0\u30eb[1]\u300d\u306a\u51e6 ... <\/p><\/div><\/dd><\/dl><\/div><\/a><\/div>\n<\/div><\/figure>\n\n\n\n

Transformer\u306e\u6210\u529f\u306e\u80cc\u5f8c\u306b\u306f\u3001\u305d\u306e\u72ec\u7279\u306a\u69cb\u9020\u304c\u3042\u308a\u307e\u3059\u3002Transformer\u306f\u6ce8\u610f\uff08attention\uff09\u6a5f\u69cb\u3092\u6838\u3068\u3057\u3066\u3044\u307e\u3059\u304c\u3001\u305d\u306e\u5468\u56f2\u306b\u306fLayerNormalization\u3084\u30d5\u30a3\u30fc\u30c9\u30d5\u30a9\u30ef\u30fc\u30c9\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u3068\u3044\u3063\u305f\u8981\u7d20\u304c\u7d44\u307f\u8fbc\u307e\u308c\u3001\u30d6\u30ed\u30c3\u30af\u69cb\u9020\uff08Transformer Block\uff09\u3092\u5f62\u6210\u3057\u3066\u3044\u307e\u3059\u3002\u3053\u306e\u30d6\u30ed\u30c3\u30af\u69cb\u9020\u306e\u512a\u308c\u305f\u8a2d\u8a08\u304c\u3001\u30e2\u30c7\u30eb\u306e\u5e30\u7d0d\u30d0\u30a4\u30a2\u30b9\u3092\u4f4e\u304f\u4fdd\u3063\u3066\u3044\u308b\u3068\u3044\u3048\u307e\u3059\u3002\u3055\u3089\u306b\u8208\u5473\u6df1\u3044\u3053\u3068\u306b\u3001\u3053\u306e\u30d6\u30ed\u30c3\u30af\u69cb\u9020\u306f\u975e\u5e38\u306b\u67d4\u8edf\u3067\u3001\u6ce8\u610f\u6a5f\u69cb\u3092\u4ed6\u306e\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u8981\u7d20\u306b\u7f6e\u304d\u63db\u3048\u3066\u3082\u3001\u9ad8\u3044\u6027\u80fd\u3092\u7dad\u6301\u3059\u308b\u3053\u3068\u304c\u53ef\u80fd\u3067\u3059\u3002\u3053\u306e\u3088\u3046\u306a\u5909\u7a2e\u306f\u300cMetaFormer[7]\u300d\u3068\u3057\u3066\u77e5\u3089\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

\n
\"\"<\/dt>

MetaFormer\u306b\u3064\u3044\u3066\u89e3\u8aac<\/p>

Transformer[1]\u306f\u3001\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u9818\u57df\u3092\u8d85\u3048\u3066\u3001\u753b\u50cf\u51e6\u7406\u306a\u3069\u306e\u5206\u91ce\u3067\u3082\u9ad8\u3044\u30d1\u30d5\u30a9\u30fc\u30de\u30f3\u30b9\u3092\u9054\u6210\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u3001\u3068\u3044\u3046\u5927\u304d\u306a\u7279\u5fb4\u3092\u6301\u3063\u3066\u3044\u307e\u3059\u3002\u4e00\u822c\u7684\u306b\u3001Transformer\u304c\u3053\u306e\u3088\u3046\u306a ... <\/p><\/div><\/dd><\/dl><\/div><\/a><\/div>\n<\/div><\/figure>\n\n\n\n

MetaFormer\u3067\u306f\u3001\u6ce8\u610f\u6a5f\u69cb\u3092\u30c8\u30fc\u30af\u30f3\u30df\u30ad\u30b5\u30fc\u3068\u3057\u3066\u8003\u3048\u3066\u3044\u307e\u3059\u3002\u30c8\u30fc\u30af\u30f3\u9593\u306e\u51e6\u7406\u304c\u53ef\u80fd\u3067\u3042\u308c\u3070\u3001\u6ce8\u610f\u6a5f\u69cb\u306f\u5fc5\u9808\u3067\u306f\u306a\u304f\u3001\u5358\u7d14\u306a\u30d7\u30fc\u30ea\u30f3\u30b0\u3060\u3051\u3067\u3082\u5341\u5206\u306a\u306e\u3067\u306f\u306a\u3044\u304b\u3001\u3068\u3044\u3046\u306e\u304cMetaFormer\u306e\u63d0\u6848\u306e\u80cc\u666f\u306b\u3042\u308a\u307e\u3059\u3002\u305f\u3060\u3057\u3001\u3053\u306e\u30a2\u30a4\u30c7\u30a3\u30a2\u306e\u691c\u8a3c\u306f\u753b\u50cf\u51e6\u7406\u306e\u5206\u91ce\u306b\u3068\u3069\u307e\u3063\u3066\u3044\u307e\u3059\u3002\u3057\u304b\u3057\u3001\u8fd1\u5e74\u306e\u7814\u7a76\u306e\u6d41\u308c\u3092\u898b\u308b\u9650\u308a\u306f\u3001\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u5206\u91ce\u3067\u3082\u6ce8\u610f\u6a5f\u69cb\u3092\u7f6e\u304d\u63db\u3048\u308b\u7814\u7a76\u306f\u591a\u304f\u884c\u308f\u308c\u3066\u304a\u308a\u3001\u540c\u69d8\u306e\u3053\u3068\u304c\u81ea\u7136\u8a00\u8a9e\u51e6\u7406\u306e\u5206\u91ce\u3067\u3082\u3044\u3048\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002\u305f\u3060\u3057\u3001\u6ce8\u610f\u6a5f\u69cb\u306e\u7f6e\u304d\u63db\u3048\u306b\u3064\u3044\u3066\u306f\u3001\u591a\u5c11\u306f\u6027\u80fd\u3068\u306e\u30c8\u30ec\u30fc\u30c9\u30aa\u30d5\u3092\u8003\u616e\u3057\u306a\u3051\u308c\u3070\u306a\u3089\u306a\u3044\u9762\u304c\u3042\u308a\u3001\u6ce8\u610f\u6a5f\u69cb\u306e\u4e07\u80fd\u3055\u306f\u63fa\u308b\u304c\u306a\u3044\u3082\u306e\u3068\u8003\u3048\u3089\u308c\u307e\u3059\u3002<\/p>\n\n\n\n

\u4ee5\u4e0a\u304b\u3089\u3001Transformer\u306e\u4e07\u80fd\u3055\u306e\u8981\u56e0\u306f\u3001\u6b21\u306e2\u3064\u304c\u5927\u304d\u304f\u95a2\u9023\u3057\u3066\u3044\u308b\u3068\u8003\u3048\u3089\u308c\u307e\u3059\u3002
\u2460\u30e2\u30c7\u30eb\u69cb\u9020\u306e\u9ad8\u3044\u62bd\u8c61\u6027
\u2461\u6ce8\u610f\u6a5f\u69cb\u306e\u9ad8\u3044\u6c4e\u7528\u6027<\/p>\n\n\n\n

\u3057\u304b\u3057\u3001Transformer\u3082\u5b8c\u74a7\u306a\u30e2\u30c7\u30eb\u3067\u306f\u3042\u308a\u307e\u305b\u3093\u3002\u7279\u306b\u5185\u7a4d\u6ce8\u610f\u306b\u95a2\u9023\u3059\u308b\u8a08\u7b97\u30b3\u30b9\u30c8\u306e\u9ad8\u3055\u3084\u3001\u5927\u91cf\u306e\u30e1\u30e2\u30ea\u8981\u4ef6\u3068\u3044\u3046\u8ab2\u984c\u304c\u6319\u3052\u3089\u308c\u307e\u3059\u3002\u73fe\u4ee3\u306e\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u3067\u306f\u3001\u3053\u308c\u3089\u306e\u554f\u984c\u306f\u907f\u3051\u3089\u308c\u306a\u3044\u5236\u7d04\u3068\u306a\u3063\u3066\u304a\u308a\u3001\u91cf\u5b50\u30b3\u30f3\u30d4\u30e5\u30fc\u30bf\u306e\u3088\u3046\u306a\u8d85\u9ad8\u901f\u8a08\u7b97\u3068\u5927\u5bb9\u91cf\u30e1\u30e2\u30ea\u304c\u5229\u7528\u3067\u304d\u308b\u74b0\u5883\u3092\u524d\u63d0\u3068\u3057\u306a\u3044\u9650\u308a\u3001\u8a08\u7b97\u30b3\u30b9\u30c8\u3092\u524a\u6e1b\u3059\u308b\u65b9\u6cd5\u3092\u6a21\u7d22\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002\u3053\u306e\u8ab2\u984c\u3092\u89e3\u6c7a\u3059\u308b\u305f\u3081\u306e\u7814\u7a76\u304c\u73fe\u5728\u3082\u6d3b\u767a\u306b\u9032\u3081\u3089\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

Attention Free Transformer<\/h3>\n\n\n\n

\u8a08\u7b97\u30b3\u30b9\u30c8\u3084\u30e1\u30e2\u30ea\u8981\u4ef6\u3068\u3044\u3063\u305f\u8ab2\u984c\u304c\u3042\u308b\u3053\u3068\u3092\u8ff0\u3079\u307e\u3057\u305f\u304c\u3001\u3053\u308c\u3089\u306e\u8ab2\u984c\u306f\u7aef\u7684\u306b\u3044\u3046\u3068\u3001\u9577\u3044\u6587\u7ae0\u3092\u52b9\u7387\u7684\u306b\u6271\u3044\u305f\u3044\u3068\u3044\u3046\u76ee\u7684\u3092\u9054\u6210\u3057\u3088\u3046\u3068\u3059\u308b\u3068\u554f\u984c\u306b\u306a\u308b\u3082\u306e\u3067\u3059\u3002\u9577\u6587\u306e\u51e6\u7406\u3092\u53ef\u80fd\u306b\u3059\u308b\u305f\u3081\u306e\u65b9\u6cd5\u3068\u3057\u3066\u3001\u4e0b\u56f3\u306b\u3082\u793a\u3055\u308c\u3066\u3044\u307e\u3059\u304c\u3001\u6ce8\u610f\u6a5f\u69cb\u306e\u6539\u826f\u3084\u5165\u529b\u306e\u6539\u826f\u304c\u8003\u3048\u3089\u308c\u307e\u3059\u3002\u6ce8\u610f\u6a5f\u69cb\u3092\u6539\u826f\u3059\u308b\u30a2\u30d7\u30ed\u30fc\u30c1\u3068\u3057\u3066\u306f\u3001\u8a08\u7b97\u30b3\u30b9\u30c8\u304c\u9ad8\u3044\u5185\u7a4d\u6ce8\u610f\u3092\u52b9\u7387\u7684\u306b\u8fd1\u4f3c\u8a08\u7b97\u3059\u308b\u624b\u6cd5\u3084\u3001\u5185\u7a4d\u6ce8\u610f\u3092\u4ed6\u306e\u624b\u6cd5\u306b\u7f6e\u304d\u63db\u3048\u308b\u65b9\u6cd5\u304c\u5b58\u5728\u3057\u307e\u3059\u3002\u307e\u305f\u3001\u5165\u529b\u306e\u6539\u826f\u3067\u306f\u3001\u5165\u529b\u30c7\u30fc\u30bf\u3092\u5727\u7e2e\u3059\u308b\u304b\u3001\u5206\u5272\u3057\u3066\u5165\u529b\u3059\u308b\u65b9\u6cd5\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n

\"\"<\/figure>\n\n\n\n

RWKV\u304c\u7279\u306b\u95a2\u9023\u3059\u308b\u306e\u306f\u3001\u5185\u7a4d\u6ce8\u610f\u3092\u4ee3\u66ff\u3059\u308b\u65b9\u6cd5\u3067\u3042\u308a\u3001\u305d\u306e\u30a2\u30d7\u30ed\u30fc\u30c1\u3092\u63a1\u7528\u3057\u305fAttention Free Transformer[8]\u304b\u3089\u5927\u304d\u306a\u5f71\u97ff\u3092\u53d7\u3051\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

Attention Free Transformer\u3068\u306f\u3001Transformer\u306eMulti-Head Attention\u3092\u3001\u30b2\u30fc\u30c6\u30a3\u30f3\u30b0\u3068WKV\uff08Weighted key Value\uff09\u306b\u3088\u308b\u6a5f\u69cb\u3067\u7f6e\u304d\u63db\u3048\u305f\u624b\u6cd5\u3067\u3059\u3002\u5185\u7a4d\u6ce8\u610f\u3092\u7121\u304f\u3057\u3001\u8907\u6570\u306e\u30d8\u30c3\u30c9\u3092\u7528\u3044\u306a\u3044\u624b\u6cd5\u306b\u3059\u308b\u3053\u3068\u3067\u5727\u5012\u7684\u306a\u8a08\u7b97\u30b3\u30b9\u30c8\u306e\u4f4e\u6e1b\u3092\u5b9f\u73fe\u3057\u307e\u3057\u305f\u3002<\/p>\n\n\n\n

\"\"<\/figure>\n\n\n\n

Attention Free Transformer\u306e\u8a73\u7d30\u306e\u8a08\u7b97\u5f0f\u306a\u3069\u304c\u6c17\u306b\u306a\u308b\u65b9\u306f\u3001\u4ee5\u4e0b\u306e\u8a18\u4e8b\u3092\u53c2\u8003\u306b\u3057\u3066\u3044\u305f\u3060\u3051\u308c\u3070\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

\n
\"\"<\/dt>

Attention Free Transformer\u306b\u3064\u3044\u3066\u89e3\u8aac<\/p>

\u306f\u3058\u3081\u306b \u672c\u8a18\u4e8b\u3067\u306f\u3001Attention Free Transformer[1]\u3068\u547c\u3070\u308c\u308b\u30e2\u30c7\u30eb\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u306b\u3064\u3044\u3066\u89e3\u8aac\u3057\u307e\u3059\u3002 \u306f\u3058\u3081\u306b\u5927\u67a0\u3092\u8aac\u660e\u3057\u307e\u3059\u3002\u8fd1\u5e74\u3067\u306fTransformer[2]\u3068\u547c ... <\/p><\/div><\/dd><\/dl><\/div><\/a><\/div>\n<\/div><\/figure>\n\n\n\n

RWKV\u306e\u767b\u5834<\/h3>\n\n\n\n

\u5185\u7a4d\u6ce8\u610f\u306f\u3001Query\u3068Key\u306e\u9593\u3067\u5168\u5165\u529b\u30c8\u30fc\u30af\u30f3\u306b\u5bfe\u3059\u308b\u5185\u7a4d\u8a08\u7b97\u304c\u4e0d\u53ef\u6b20\u3060\u3063\u305f\u305f\u3081\u3001\u65b0\u3057\u3044\u30c8\u30fc\u30af\u30f3\u304c\u751f\u6210\u3055\u308c\u308b\u305f\u3073\u306b\u3001\u305d\u308c\u3092\u542b\u3080\u904e\u53bb\u306e\u5168\u3066\u306e\u30c8\u30fc\u30af\u30f3\u5217\u3092\u5165\u529b\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u3001\u305d\u308c\u304c\u8a08\u7b97\u30b3\u30b9\u30c8\u304c\u304b\u3055\u3080\u8981\u56e0\u3068\u306a\u3063\u3066\u3044\u307e\u3057\u305f\u3002Attention Free Transformer\u306e\u767b\u5834\u306b\u3088\u308a\u3001Query\u3068Key\u306e\u5185\u7a4d\u8a08\u7b97\u3092\u5fc5\u8981\u3068\u3057\u306a\u3044\u65b0\u3057\u3044\u6ce8\u610f\u6a5f\u69cb\u3001Weighted Key Value\uff08WKV\uff09\u65b9\u5f0f\u304c\u63d0\u6848\u3055\u308c\u3001\u5185\u7a4d\u306e\u8a08\u7b97\u304c\u4e0d\u8981\u306b\u306a\u308a\u8a08\u7b97\u30b3\u30b9\u30c8\u3092\u5927\u5e45\u306b\u6539\u5584\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3057\u305f\u3002\u3057\u304b\u3057\u3001Attention Free Transformer\u3067\u306f\u3001\u307e\u3060\u904e\u53bb\u306e\u3059\u3079\u3066\u306e\u60c5\u5831\u3092\u5165\u529b\u3059\u308b\u5fc5\u8981\u304c\u3042\u3063\u305f\u305f\u3081\u3001RWKV\u3067\u306f\u305d\u308c\u3092\u6539\u826f\u3057\u3001WKV\u306b\u518d\u5e30\u7684\u6027\u8cea\u3092\u53d6\u308a\u5165\u308c\u308b\u3053\u3068\u3067\u3001\u4ee5\u524d\u306e\u5168\u3066\u306e\u60c5\u5831\u3092\u5165\u529b\u3059\u308b\u3053\u3068\u306a\u304f\u3001\u52b9\u7387\u7684\u306b\u51e6\u7406\u3092\u884c\u3048\u308bReceptance Weighted Key Value\u65b9\u5f0f\u3092\u63d0\u6848\u3057\u307e\u3057\u305f\u3002<\/p>\n\n\n\n

\u518d\u5e30\u6027\u3092\u5c0e\u5165\u3059\u308b\u3053\u3068\u3067\u3001\u4e26\u5217\u6f14\u7b97\u304c\u56f0\u96e3\u306b\u306a\u308b\u3068\u3044\u3046\u7591\u554f\u304c\u6d6e\u304b\u3076\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002\u78ba\u304b\u306b\u3001\u518d\u5e30\u7684\u306a\u90e8\u5206\u306b\u95a2\u3057\u3066\u306f\u6642\u9593\u8ef8\u4e0a\u3067\u306e\u30b7\u30ea\u30a2\u30eb\u306a\u8a08\u7b97\u304c\u5fc5\u9808\u3067\u3059\u3002\u3057\u304b\u3057\u306a\u304c\u3089\u3001\u3053\u306e\u5f71\u97ff\u3092\u6700\u5c0f\u9650\u306b\u6291\u3048\u308b\u30c6\u30af\u30cb\u30c3\u30af\u304c\u3042\u308a\u307e\u3059\u3002\u5177\u4f53\u7684\u306b\u306f\u3001\u518d\u5e30\u7684\u306a\u30eb\u30fc\u30d7\u3092\u6301\u3064\u69cb\u9020\u3092\u3001\u3067\u304d\u308b\u3060\u3051\u5c0f\u3055\u306a\u5185\u90e8\u30e1\u30ab\u30cb\u30ba\u30e0\u306b\u9650\u5b9a\u3059\u308b\u3053\u3068\u3067\u3059\u3002\u3053\u306e\u5de5\u592b\u306b\u3088\u308a\u3001\u518d\u5e30\u7684\u306a\u90e8\u5206\u306e\u524d\u5f8c\u3067\u306e\u8a08\u7b97\u306f\u4e26\u5217\u7684\u306b\u9ad8\u901f\u306b\u5b9f\u884c\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002RWKV\u3067\u306f\u3001\u518d\u5e30\u6027\u3092WKV\u3060\u3051\u306b\u9650\u5b9a\u3057\u3066\u3044\u308b\u305f\u3081\u3001\u305d\u308c\u4ee5\u5916\u306e\u90e8\u5206\u306f\u4e26\u5217\u3067\u9ad8\u901f\u306b\u8a08\u7b97\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u308b\u306e\u3067\u3001\u5b66\u7fd2\u30d5\u30a7\u30fc\u30ba\u3067\u306f\u4e26\u5217\u8a08\u7b97\u306e\u30e1\u30ea\u30c3\u30c8\u3092\u4eab\u53d7\u3057\u3064\u3064\u3001\u63a8\u8ad6\u6642\u306b\u306fRNN\u306e\u3088\u3046\u306a\u52b9\u7387\u7684\u306a\u8a08\u7b97\u304c\u53ef\u80fd\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

<\/i>

RWKV\u306e\u30dd\u30a4\u30f3\u30c8<\/mark> <\/h4><\/div>
\n
    \n
  • RNN\u3068Transformer\u306e\u5229\u70b9\u3092\u7d44\u307f\u5408\u308f\u305b\u305f\u30e2\u30c7\u30eb\n
      \n
    • \u5b66\u7fd2\u6642\u306f\u51e6\u7406\u3092\u4e26\u5217\u5316\u3057\u3066\u9ad8\u901f\u306b\u5b9f\u884c\u53ef\u80fd<\/li>\n\n\n\n
    • \u63a8\u8ad6\u6642\u306f\u73fe\u5728\u306e\u5165\u529b\u3068\u72b6\u614b\u5909\u6570\u3092\u7528\u3044\u3066\u5c11\u306a\u3044\u8a08\u7b97\u30b3\u30b9\u30c8\u3067\u5b9f\u884c\u53ef\u80fd<\/li>\n<\/ul>\n<\/li>\n\n\n\n
    • \u4e0a\u8a18\u3092\u5b9f\u73fe\u3059\u308b\u305f\u3081\u306bRWKV Attention\u3092\u5c0e\u5165<\/li>\n<\/ul>\n<\/div><\/div>\n\n\n\n

      RWKV\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3<\/h2>\n\n\n\n

      RWKV\u306e\u57fa\u672c\u69cb\u9020<\/h3>\n\n\n\n

      \u307e\u305a\u306f\u3001RWKV\u306e\u57fa\u672c\u69cb\u9020\u304b\u3089\u78ba\u8a8d\u3057\u3066\u307f\u307e\u3057\u3087\u3046\u3002\u4e0b\u56f3\u5de6\u306bTransformer Block\u3001\u53f3\u306bRWKV Block\u3092\u793a\u3057\u307e\u3057\u305f\u3002\u898b\u3066\u3082\u3089\u3046\u3068\u5206\u304b\u308b\u3068\u601d\u3044\u307e\u3059\u304c\u3001RWKV\u306e\u57fa\u672c\u69cb\u9020\u306fTransformer\u3068\u540c\u69d8\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002\u9055\u3044\u306f\u3001Multi-Head Attention\u304cTime Mixing\u306b\u3001Feed Forward\u304cChannel Mixinf\u306b\u66ff\u308f\u3063\u3066\u3044\u308b\u70b9\u3068\u3001\u305d\u308c\u305e\u308c\u518d\u5e30\u69cb\u9020\u3092\u6301\u3064\u70b9\u3067\u3059\u3002<\/p>\n\n\n\n

      \"\"<\/figure>\n\n\n\n

      RWKV\u3092\u7528\u3044\u3066\u7cfb\u5217\u51e6\u7406\u3092\u884c\u3046\u969b\u306e\u60c5\u5831\u306e\u6d41\u308c\u3092\u3001\u6642\u9593\u8ef8\u65b9\u5411\u306b\u5c55\u958b\u3057\u305f\u518d\u5e30\u69cb\u9020\u3067\u8003\u3048\u3066\u307f\u307e\u3057\u3087\u3046\u3002\u4ee5\u4e0b\u306e\u56f3\u3067\u306f\u3001RWKV\u3092\u4f7f\u3063\u305f\u6587\u7ae0\u751f\u6210\u306e\u4e00\u4f8b\u3092\u793a\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

      RWKV\u3092\u4f7f\u7528\u3059\u308b\u3068\u3001\u5404\u30b9\u30c6\u30c3\u30d7\u306b\u5fc5\u8981\u306a\u5165\u529b\u306f1\u30c8\u30fc\u30af\u30f3\u3060\u3051\u3067\u3059\u3002\u3057\u305f\u304c\u3063\u3066\u3001\u300cMy\u300d\u3001\u300cname\u300d\u3001\u300cis\u300d\u3068\u3044\u3046\u3088\u3046\u306b\u3001\u4e00\u3064\u305a\u3064\u9806\u306b\u5165\u529b\u3057\u3066\u3044\u3051\u3070\u826f\u3044\u306e\u3067\u3059\u3002
      \u203b\u4e00\u65b9\u3001Transformer\u3092\u4f7f\u7528\u3059\u308b\u5834\u5408\u3001\u4f8b\u3048\u3070\u300cMy\u300d\u306e\u5f8c\u306b\u300cMy name\u300d\u3001\u305d\u306e\u5f8c\u306b\u300cMy name is\u300d\u3068\u3001\u524d\u306e\u5165\u529b\u3082\u542b\u3081\u3066\u5165\u529b\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3057\u305f\u3002<\/p>\n\n\n\n

      RWKV\u306e\u518d\u5e30\u7684\u306a\u69cb\u9020\u306b\u306f\u3001\u300cToken shift\u300d\u3068\u300cStates\u300d\u306e\u4e8c\u3064\u306e\u8981\u7d20\u304c\u3042\u308a\u307e\u3059\u3002\u300cToken shift\u300d\u3068\u306f\u3001\u4e00\u3064\u524d\u306e\u5165\u529b\u30c8\u30fc\u30af\u30f3\u3092\u4fdd\u6301\u3057\u3001\u6b21\u306e\u6642\u523b\u3067\u306e\u51e6\u7406\u306b\u4f7f\u7528\u3059\u308b\u305f\u3081\u306e\u60c5\u5831\u306e\u6d41\u308c\u3092\u6307\u3057\u307e\u3059\u3002\u300cStates\u300d\u306f\u3001WKV\u306e\u5185\u90e8\u72b6\u614b\u3092\u793a\u3057\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n

      \n
      \"\"
      [3]\u3088\u308a\u5f15\u7528<\/figcaption><\/figure><\/div>\n\n\n

      Time Mixing<\/h3>\n\n\n\n

      Time Mixing\u306e\u6a5f\u80fd\u306b\u3064\u3044\u3066\u3001\u8a73\u3057\u304f\u3054\u7406\u89e3\u3044\u305f\u3060\u304f\u305f\u3081\u3001\u307e\u305a\u6570\u5f0f\u306e\u89e3\u8aac\u304b\u3089\u59cb\u3081\u3066\u3044\u304d\u307e\u3059\u3002\u4ee5\u4e0b\u304cTime Mixing\u306e\u6570\u5f0f\u3067\u3059\u3002<\/p>\n\n\n\n

      $$\\begin{eqnarray}
      \\boldsymbol{r}_t &=& \\boldsymbol{W_r}\\cdot (\\mu_r \\boldsymbol{x}_t + (1-\\mu_r)\\boldsymbol{x}_{t-1}) \\\\
      \\boldsymbol{k}_t &=& \\boldsymbol{W_k}\\cdot (\\mu_k \\boldsymbol{x}_t + (1-\\mu_k)\\boldsymbol{x}_{t-1}) \\\\
      \\boldsymbol{v}_t &=& \\boldsymbol{W_v}\\cdot (\\mu_v \\boldsymbol{x}_t + (1-\\mu_v)\\boldsymbol{x}_{t-1}) \\\\
      \\boldsymbol{wkv}_t &=& \\frac{\\sum_{i=1}^{t-1}\\exp(-(t-1-i)w + \\boldsymbol{k}_i)\\boldsymbol{v}_i + \\exp(u + \\boldsymbol{k}_t)\\boldsymbol{v}_t}{\\sum_{i=1}^{t-1}\\exp(-(t-1-i)w + \\boldsymbol{k}_i) + \\exp(u + \\boldsymbol{k}_t)}\\\\
      \\boldsymbol{o}_t &=& \\boldsymbol{W_o}\\cdot (\\sigma (r_t)\\odot \\boldsymbol{wkv}_t)
      \\end{eqnarray}$$<\/p>\n\n\n\n

      Time Mixing\u306f\u3001RWKV\u306e\u30d9\u30fc\u30b9\u3068\u306a\u3063\u305fAttention Free Transformer\uff08\u4ee5\u964d\u3001AFT\uff09\u3068\u540c\u69d8\u306b\u3001Multi-Head Attention\u3092\u4ee3\u66ff\u3059\u308b\u305f\u3081\u3001\u7dda\u5f62\u5c64\u3082\u7d44\u307f\u8fbc\u307e\u308c\u3066\u3044\u307e\u3059\u3002RWKV\u3092\u69cb\u7bc9\u3059\u308bTime Mixing\u306e\u5165\u529b\u306f\u3001AFT\u3068\u306f\u7570\u306a\u308a\u3001\u904e\u53bb\u306e\u30c8\u30fc\u30af\u30f3\u3092\u542b\u3093\u3067\u3044\u307e\u3059\u304c\u3001\u5fc5\u8981\u306a\u306e\u306f\u73fe\u5728\u306e\u5165\u529b\\(\\boldsymbol{x}_t\\)\u3068\u76f4\u524d\u306e\u5165\u529b\\(\\boldsymbol{x}_{t-1}\\)\u306e\u307f\u3067\u3059\u3002\u305d\u3057\u3066\u3001\u3053\u308c\u3089\u306f\u76f4\u63a5\u7dda\u5f62\u5c64\u3067\u5909\u63db\u3055\u308c\u308b\u306e\u3067\u306f\u306a\u304f\u3001\u30d1\u30e9\u30e1\u30fc\u30bf\\(\\mu\\)\u3092\u4f7f\u3063\u3066\u7dda\u5f62\u7d50\u5408\u3055\u308c\u3001\u305d\u306e\u5f8c\u5909\u63db\u3055\u308c\u307e\u3059\uff08\\(\\boldsymbol{r}_t, \\boldsymbol{k}_t, \\boldsymbol{v}_t\\)\uff09\u3002 \u3053\u3053\u307e\u3067\u306e\u8aac\u660e\u3067\u306f\u3001\u6a19\u6e96\u306eAttention\u3068\u306f\u5c11\u3057\u7570\u306a\u308b\u8a08\u7b97\u3092\u3057\u3066\u3044\u308b\u306e\u304c\u5206\u304b\u308a\u307e\u3059\u304c\u3001\u5927\u304d\u306a\u969c\u58c1\u306a\u304f\u7406\u89e3\u3067\u304d\u308b\u304b\u3068\u601d\u3044\u307e\u3059\u3002\u7d9a\u3044\u3066\u3001\\(\\boldsymbol{wkv}_t\\)\u306e\u8a08\u7b97\u8a73\u7d30\u3092\u898b\u3066\u3044\u304d\u307e\u3057\u3087\u3046\u3002\u3053\u306e\u8a08\u7b97\u306f\u5c11\u3057\u8907\u96d1\u3067\u3059\u304c\u3001RWKV\u306e\u4e2d\u5fc3\u7684\u306a\u90e8\u5206\u3067\u3059\u3002\u3055\u3066\u3001\\(\\boldsymbol{wkv}_t\\)\u306e\u8a08\u7b97\u5f0f\u3092\u898b\u308b\u3068\u3001\u904e\u53bb\u306e\\(\\boldsymbol{k}\\)\u3068\\(\\boldsymbol{v}\\)\u3092\u4f7f\u7528\u3057\u3066\u3044\u308b\u305f\u3081\u3001\u4e00\u898b\u3001\u518d\u5e30\u7684\u306a\u8a08\u7b97\u304c\u306a\u3044\u3088\u3046\u306b\u611f\u3058\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002\u3057\u304b\u3057\u3001\u5b9f\u969b\u306b\u306f\u3001\u5f8c\u306b\u7d39\u4ecb\u3059\u308b\u72b6\u614b\u5909\u6570\\(\\boldsymbol{\\alpha}\\)\u3068\\(\\boldsymbol{\\beta}\\)\u3092\u7528\u3044\u3066\u3001RNN\u306e\u3088\u3046\u306b\u524d\u306e\u72b6\u614b\u3092\u7528\u3044\u3066\u52b9\u7387\u7684\u306b\u8a08\u7b97\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002 \\(\\boldsymbol{\\alpha}\\)\u3068\\(\\boldsymbol{\\beta}\\)\u306e\u5b9a\u7fa9\u306f\u6b21\u306e\u3088\u3046\u306b\u306a\u308a\u307e\u3059\u3002<\/p>\n\n\n\n

      $$\\begin{eqnarray}
      \\boldsymbol{\\alpha}_t &=& \\sum_{i=1}^{t}\\exp(-(t-i)w + \\boldsymbol{k}_i)\\boldsymbol{v}_i \\\\
      \\boldsymbol{\\beta}_t &=& \\sum_{i=1}^{t}\\exp(-(t-i)w + \\boldsymbol{k}_i
      \\end{eqnarray}$$<\/p>\n\n\n\n

      \u3053\u308c\u306b\u3088\u308a\u3001\u4ee5\u4e0b\u306e\u6f38\u5316\u5f0f\u3067\u305d\u308c\u305e\u308c\u306e\u5909\u6570\u3092\u8868\u3059\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n

      $$\\begin{eqnarray}
      \\boldsymbol{\\alpha}_t &=& \\exp(-w)\\boldsymbol{\\alpha}_{t-1} + \\exp(\\boldsymbol{k}_t)\\boldsymbol{v}_t\\\\
      \\boldsymbol{\\beta}_t &=& \\exp(-w)\\boldsymbol{\\beta}_{t-1} + \\exp(\\boldsymbol{k}_t)
      \\end{eqnarray}$$<\/p>\n\n\n\n

      \u3053\u306e\u3068\u304d\u3001\\(\\boldsymbol{wkv}\\)\u306f\u72b6\u614b\u5909\u6570\u3092\u7528\u3044\u3066\u4ee5\u4e0b\u306e\u3088\u3046\u8a08\u7b97\u3055\u308c\u307e\u3059\u3002<\/p>\n\n\n\n

      $$
      \\boldsymbol{wkv}_t = \\frac{\\boldsymbol{\\alpha}_{t-1} + \\exp(u + \\boldsymbol{k}_t)\\boldsymbol{v}_t}{\\boldsymbol{\\beta}_{t-1} + \\exp(u + \\boldsymbol{k}_t)}
      $$<\/p>\n\n\n\n

      \u72b6\u614b\u5909\u6570\\(\\boldsymbol{\\alpha}\\)\u3068\\(\\boldsymbol{\\beta}\\)\u3092\u7528\u3044\u308b\u3053\u3068\u3067\u3001\u904e\u53bb\u306e\u60c5\u5831\u3092\u518d\u8a08\u7b97\u3059\u308b\u3053\u3068\u306a\u304f\u52b9\u7387\u7684\u306b\u8a08\u7b97\u3067\u304d\u307e\u3059\u3002\u3053\u306e\u7279\u5fb4\u304c\u3001RNN\u306e\u3088\u3046\u306a\u9ad8\u901f\u306a\u63a8\u8ad6\u3084\u3001\u9577\u3044\u30c6\u30ad\u30b9\u30c8\u306e\u30b7\u30fc\u30b1\u30f3\u30b9\u3092\u6271\u3046\u3053\u3068\u3092\u53ef\u80fd\u306b\u3057\u3066\u3044\u307e\u3059\u3002 \u3053\u308c\u3089\u306e\u6570\u5f0f\u3092\u901a\u3057\u3066\u3001RWKV\u306eTime Mixing\u306e\u6838\u5fc3\u3092\u7406\u89e3\u3057\u3066\u3044\u305f\u3060\u3051\u305f\u304b\u3068\u601d\u3044\u307e\u3059\u3002\u3055\u3089\u306b\u6df1\u3044\u7406\u89e3\u306e\u305f\u3081\u3001\u6570\u5f0f\u3092\u56f3\u3067\u8868\u73fe\u3057\u3066\u307f\u307e\u3057\u305f\u3002\u56f3\u3092\u53c2\u8003\u306b\u3001\u3082\u3046\u4e00\u5ea6\u6570\u5f0f\u3092\u78ba\u8a8d\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002<\/p>\n\n\n\n

      \"\"<\/figure>\n\n\n\n

      Channel Mixing<\/h3>\n\n\n\n

      RWKV\u306e\u3082\u3046\u4e00\u3064\u306e\u91cd\u8981\u306a\u4ed5\u7d44\u307f\u3067\u3042\u308b\u3001Channel Mixing\u306b\u3064\u3044\u3066\u8aac\u660e\u3057\u307e\u3059\u3002<\/p>\n\n\n\n

      Channel Mixing\u306f\u4ee5\u4e0b\u306e\u6570\u5f0f\u3067\u8868\u3055\u308c\u307e\u3059\u3002<\/p>\n\n\n\n

      $$\\begin{eqnarray}
      r_t &=& \\boldsymbol{W_r}\\cdot (\\mu_r x_t + (1-\\mu_r)x_{t-1}) \\\\
      k_t &=& \\boldsymbol{W_k}\\cdot (\\mu_k x_t + (1-\\mu_k)x_{t-1}) \\\\
      o_t &=& \\sigma (r_t) \\odot (\\boldsymbol{W_v} \\cdot \\max (k_t, 0)^2)
      \\end{eqnarray}$$<\/p>\n\n\n\n

      \u3053\u306e\u6570\u5f0f\u306fTime Mixing\u3088\u308a\u3082\u975e\u5e38\u306b\u30b7\u30f3\u30d7\u30eb\u306a\u5f62\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002\u5165\u529b\u3068\u3057\u3066\u306f\u6642\u523b\\(t\\)\u3068\u6642\u523b\\(t-1\\)\u306e2\u3064\u3092\u7528\u3044\u3001\u305d\u308c\u3089\u3092\u30d1\u30e9\u30e1\u30fc\u30bf\\(\\mu\\)\u306b\u3088\u308a\u7dda\u5f62\u7d50\u5408\u3057\u3066\u51e6\u7406\u3057\u307e\u3059\u3002\u6570\u5f0f\u306b\u306f\u3001\\(\\boldsymbol{W_r}, \\boldsymbol{W_k}, \\boldsymbol{W_v}\\)\u306e3\u3064\u306e\u91cd\u307f\u884c\u5217\u304c\u542b\u307e\u308c\u3066\u3044\u307e\u3059\u304c\u3001\u3053\u308c\u3089\u306fTime Mixing\u306e\u91cd\u307f\u3068\u306f\u5171\u6709\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u56f3\u89e3\u3059\u308b\u3068\u3001\u53f3\u5074\u306e\\(\\boldsymbol{W_k}\\)\u3084\\( \\boldsymbol{W_v}\\)\u306e\u3042\u308b\u30d1\u30b9\u306f\u3001\u4e2d\u9593\u5c64\u306e\u6d3b\u6027\u5316\u95a2\u6570\u304c\\(\\max(\\boldsymbol{k}, 0)^2\\)\u3001\u51fa\u529b\u5c64\u306e\u6d3b\u6027\u5316\u95a2\u6570\u304c\u6052\u7b49\u95a2\u6570\u306e3\u5c64\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u3068\u8003\u3048\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u305d\u306e\u7d50\u679c\u3092\u5de6\u5074\u306e\u5217\u3001\u3064\u307e\u308a2\u5c64\u30cb\u30e5\u30fc\u30e9\u30eb\u30cd\u30c3\u30c8\u30ef\u30fc\u30af\u3067\u30b2\u30fc\u30c6\u30a3\u30f3\u30b0\u3059\u308b\u5f62\u306b\u306a\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n

      \n
      \"\"<\/figure><\/div>\n\n\n

      \u63a8\u8ad6\u6642\u306b\u9ad8\u901f\u5316\u304c\u53ef\u80fd\u306a\u7406\u7531<\/h3>\n\n\n\n

      RWKV\u304cRNN\u306e\u7279\u6027\u3092\u6301\u3064\u3053\u3068\u3067\u3001\u63a8\u8ad6\u306e\u969b\u306b\u975e\u5e38\u306b\u9ad8\u901f\u306b\u51e6\u7406\u3059\u308b\u3053\u3068\u304c\u53ef\u80fd\u3068\u306a\u3063\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

      \u5177\u4f53\u7684\u306b\u3001\u901a\u5e38\u306eTransformer\u3092\u4f7f\u3063\u3066\u300cMy name is Bob\u300d\u3068\u3044\u3046\u6587\u7ae0\u3092\u51e6\u7406\u3059\u308b\u5834\u9762\u3092\u8003\u3048\u3066\u307f\u307e\u3057\u3087\u3046\u3002\u3053\u306e\u5834\u5408\u3001<\/p>\n\n\n\n

      \u6642\u523b1: \uff1c\u5165\u529b\uff1eMy\uff1c\u51fa\u529b\uff1ename
      \u6642\u523b2: \uff1c\u5165\u529b\uff1eMy name\uff1c\u51fa\u529b\uff1eis
      \u6642\u523b3: \uff1c\u5165\u529b\uff1eMy name is\uff1c\u51fa\u529b\uff1eBob<\/p>\n\n\n\n

      \u3068\u3044\u3046\u3088\u3046\u306b\u3001\u5404\u6642\u523b\u3067\u306e\u5358\u8a9e\u30d9\u30af\u30c8\u30eb\u3092\u751f\u6210\u3057\u3001\u305d\u308c\u3092\u542b\u3080\u904e\u53bb\u5168\u3066\u306e\u30c8\u30fc\u30af\u30f3\u3092\u518d\u5ea6\u5165\u529b\u3068\u3057\u3066\u6b21\u306e\u5358\u8a9e\u3092\u4e88\u6e2c\u3059\u308b\u305f\u3081\u306e\u51e6\u7406\u3092\u7e70\u308a\u8fd4\u3059\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\u3002<\/p>\n\n\n\n

      \u3057\u304b\u3057\u3001RWKV\u306e\u5834\u5408\u306f\u904e\u53bb\u306e\u72b6\u614b\u3092\u4fdd\u6301\u3057\u3066\u3044\u308b\u306e\u3067\u3001\u6b21\u306e\u3088\u3046\u306a\u30b7\u30f3\u30d7\u30eb\u306a\u51e6\u7406\u304c\u53ef\u80fd\u3067\u3059\u3002<\/p>\n\n\n\n

      \u6642\u523b1: \uff1c\u5165\u529b\uff1eMy\uff1c\u51fa\u529b\uff1ename
      \u6642\u523b2: \uff1c\u5165\u529b\uff1ename\uff1c\u51fa\u529b\uff1eis
      \u6642\u523b3: \uff1c\u5165\u529b\uff1eis\uff1c\u51fa\u529b\uff1eBob<\/p>\n\n\n\n

      \u3053\u306e\u65b9\u5f0f\u306f\u3001RNN\u306e\u30e1\u30ea\u30c3\u30c8\u3092Transformer\u306b\u53d6\u308a\u5165\u308c\u305f\u3082\u306e\u3067\u3001\u51e6\u7406\u306e\u52b9\u7387\u3092\u5927\u304d\u304f\u5411\u4e0a\u3055\u305b\u3066\u3044\u307e\u3059\u3002\u305d\u306e\u7d50\u679c\u3001\u4f8b\u3048\u3070Raspberry Pi\u306e\u3088\u3046\u306a\u30c7\u30d0\u30a4\u30b9\u3067\u3082\u3001RWKV\u3092\u52d5\u304b\u3059\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002<\/p>\n\n\n\n

      \u5b66\u7fd2\u6642\u306b\u4e26\u5217\u5316\u304c\u53ef\u80fd\u306a\u7406\u7531<\/h3>\n\n\n\n

      \u63a8\u8ad6\u6642\u306b\u8a08\u7b97\u30b3\u30b9\u30c8\u304c\u5927\u5e45\u306b\u524a\u6e1b\u3055\u308c\u308b\u3053\u3068\u3092\u8aac\u660e\u3057\u307e\u3057\u305f\u304c\u3001\u300c\u3067\u306f\u5b66\u7fd2\u6642\u306f\u3069\u3046\u306a\u306e\u304b\uff1f\u300d\u3068\u3044\u3046\u7591\u554f\u304c\u6d6e\u304b\u3073\u307e\u3059\u3088\u306d\u3002\u5b9f\u306f\u3001\u5b66\u7fd2\u6642\u306fRWKV\u306f\u4e26\u5217\u51e6\u7406\u304c\u53ef\u80fd\u3067\u3042\u308a\u3001\u3053\u308c\u306fTransformer\u306e\u5927\u304d\u306a\u5229\u70b9\u3067\u3042\u308b\u4e26\u5217\u8a08\u7b97\u80fd\u529b\u3092\u7d99\u627f\u3057\u3066\u3044\u308b\u306e\u3067\u3059\u3002<\/p>\n\n\n\n

      \u5177\u4f53\u7684\u306a\u65b9\u6cd5\u3092\u8aac\u660e\u3059\u308b\u305f\u3081\u3001\u300cMy name is Bob\u300d\u3068\u3044\u3046\u5165\u529b\u30c6\u30ad\u30b9\u30c8\u3092\u4f8b\u306b\u3068\u3063\u3066\u8003\u3048\u3066\u307f\u307e\u3057\u3087\u3046\u3002<\/p>\n\n\n\n

      \u63a8\u8ad6\u6642\u3068\u306f\u9055\u3044\u3001\u5b66\u7fd2\u6642\u306b\u306f\u524d\u306e\u6642\u523b\u306e\u51e6\u7406\u3092\u6700\u5f8c\u307e\u3067\u5f85\u3064\u3053\u3068\u306a\u304f\u3001\u300cMy name is\u300d\u3068\u3044\u3046\u5165\u529b\u304b\u3089\u300cname is Bob\u300d\u3068\u3044\u3046\u51fa\u529b\u3092\u751f\u6210\u3059\u308b\u3053\u3068\u304c\u3067\u304d\u307e\u3059\u3002\u5177\u4f53\u7684\u306b\u306f\u3001\u300cMy name is\u300d\u306b\u3064\u3044\u3066Time Mixing\u306e\u76f4\u524d\u307e\u3067\u30d0\u30c3\u30c1\u51e6\u7406\u3057\u3066\u3057\u307e\u3044\u307e\u3059\uff08\u4e0b\u56f3\u306b\u304a\u3044\u3066\u9752\u8272\u77e2\u5370\uff09\u3002Time Mixing\u306b\u3064\u3044\u3066\u306f\u3001\u72b6\u614b\u3092\u6301\u3061\u307e\u3059\u304b\u3089\u30b7\u30ea\u30a2\u30eb\u306b\u8a08\u7b97\u3059\u308b\u5fc5\u8981\u304c\u3042\u308a\u307e\u3059\uff08\u4e0b\u56f3\u306b\u304a\u3044\u3066\u9ec4\u8272\u77e2\u5370\uff09\u3002\u3068\u306f\u3044\u3048\u3001\u305d\u308c\u306fTime Mixing\u306e\u90e8\u5206\u3060\u3051\u3067\u6e08\u3080\u308f\u3051\u3067\u3059\u3002\u305d\u308c\u4ee5\u5916\u306e\u90e8\u5206\u306f\u4e00\u6c17\u306b\u30d0\u30c3\u30c1\u51e6\u7406\u3057\u3066\u3057\u307e\u3044\u307e\u3059\uff08\u4e0b\u56f3\u306b\u304a\u3044\u3066\u9752\u8272\u77e2\u5370\uff09\u3002 \u8981\u3059\u308b\u306b\u3001\u72b6\u614b\u30d9\u30af\u30c8\u30eb\u3067\u3042\u308b\\(\\boldsymbol{\\alpha}\\)\u3068\\(\\boldsymbol{\\beta}\\)\u3092\u9664\u3044\u3066\u306f\u3001\u5b66\u7fd2\u30c7\u30fc\u30bf\u3092\u30d0\u30c3\u30c1\u5316\u3057\u3066\u4e26\u5217\u306b\u8a08\u7b97\u3059\u308b\u3053\u3068\u304c\u53ef\u80fd\u306a\u306e\u3067\u3059\u3002<\/p>\n\n\n

      \n
      \"\"<\/figure><\/div>\n\n\n

      \u307e\u3068\u3081<\/h2>\n\n\n\n

      RWKV\u306f\u3001\u8a55\u4fa1\u6307\u6a19\u4e0a\u3067\u306f\u4ed6\u306eTransformer\u30d9\u30fc\u30b9\u306e\u624b\u6cd5\u3068\u4e26\u3076\u6027\u80fd\u3092\u6301\u3064\u3068\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u3057\u304b\u3057\u3001\u5185\u7a4d\u6ce8\u610f\u3068\u7570\u306a\u308a\u3001\u904e\u53bb\u306e\u60c5\u5831\u3092\u7279\u5b9a\u3057\u3066\u632f\u308a\u8fd4\u308b\u80fd\u529b\u306b\u5236\u7d04\u304c\u3042\u308b\u305f\u3081\u3001\u904e\u53bb\u306e\u660e\u78ba\u306a\u60c5\u5831\u3078\u306e\u30a2\u30af\u30bb\u30b9\u304c\u5fc5\u8981\u306a\u30b7\u30c1\u30e5\u30a8\u30fc\u30b7\u30e7\u30f3\u3067\u306f\u3001Transformer\u306b\u6bd4\u3079\u3066\u6027\u80fd\u304c\u4f4e\u4e0b\u3059\u308b\u53ef\u80fd\u6027\u304c\u6307\u6458\u3055\u308c\u3066\u3044\u307e\u3059\u3002\u3053\u306e\u70b9\u306f\u3001\u8ad6\u6587\u3067\u3082\u78ba\u8a8d\u3055\u308c\u3066\u304a\u308a\u3001\u4eca\u5f8c\u306e\u7814\u7a76\u306e\u8ab2\u984c\u3068\u3057\u3066\u6319\u3052\u3089\u308c\u3066\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

      \u3053\u306e\u305f\u3081\u3001\u8907\u96d1\u306a\u30bf\u30b9\u30af\u3001\u4f8b\u3048\u3070\u5165\u8a66\u554f\u984c\u3092\u89e3\u304f\u3088\u3046\u306a\u30b7\u30c1\u30e5\u30a8\u30fc\u30b7\u30e7\u30f3\u3067\u306fRWKV\u306e\u6027\u80fd\u306b\u671f\u5f85\u304c\u6301\u3066\u306a\u3044\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002\u4e00\u65b9\u3067\u3001\u7d44\u307f\u8fbc\u307f\u6a5f\u5668\u306bRWKV\u3092\u5c0e\u5165\u3057\u3066\u4f1a\u8a71\u30ed\u30dc\u30c3\u30c8\u3092\u4f5c\u308b\u306a\u3069\u3001\u7c21\u5358\u306a\u65e5\u5e38\u4f1a\u8a71\u7a0b\u5ea6\u306e\u5229\u7528\u3067\u3042\u308c\u3070\u3001\u5229\u7528\u3067\u304d\u308b\u304b\u3082\u3057\u308c\u307e\u305b\u3093\u3002<\/p>\n\n\n\n

      \u3067\u3059\u304c\u3001\u73fe\u72b6\u3060\u3068RWKV\u306f\u3001\u3042\u307e\u308a\u9ad8\u3044\u6027\u80fd\u3068\u306f\u8a00\u3048\u306a\u3044\u5370\u8c61\u3092\u53d7\u3051\u307e\u3059\u3002\u6027\u80fd\u9762\u3067\u3044\u3046\u3068\u3001\u591a\u5c11\u306e\u9045\u5ef6\u3084\u30c8\u30fc\u30af\u30f3\u6570\u306b\u5236\u7d04\u304c\u3042\u3063\u3066\u3082\u5185\u7a4d\u6ce8\u610f\u30d9\u30fc\u30b9\u306eTransformer\u306e\u65b9\u304c\u9b45\u529b\u7684\u306b\u611f\u3058\u307e\u3059\u3002<\/p>\n\n\n\n

      \u3057\u304b\u3057\u3001RWKV\u306b\u306f\u5927\u304d\u306a\u30dd\u30c6\u30f3\u30b7\u30e3\u30eb\u304c\u3042\u308b\u3068\u611f\u3058\u3066\u3044\u307e\u3059\u3002\u3067\u3059\u306e\u3067\u3001\u4eca\u5f8c\u306e\u52d5\u5411\u306b\u6ce8\u76ee\u3057\u305f\u3044\u3068\u601d\u3044\u307e\u3059\u3002<\/p>\n\n\n\n

      \u53c2\u8003\u6587\u732e<\/h2>\n\n\n\n

      [1] Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser, and Illia Polosukhin, \"Attention is all you need,\" in Proc. NeurIPS<\/em>, 2017.
      [2] David E. Rumelhart and James L. McClelland, \u201cLearning Internal Representations by Error Propagation,\u201d in Parallel Distributed Processing: Explorations in the Microstructure of Cognition: Foundations<\/em>, 1987.
      [3] Bo Peng, Eric Alcaide, Quentin Anthony, Alon Albalak, Samuel Arcadinho, Huanqi Cao, Xin Cheng, Michael Chung, Matteo Grella, Kranthi Kiran GV, Xuzheng He, Haowen Hou, Przemyslaw Kazienko, Jan Kocon, Jiaming Kong, Bartlomiej Koptyra, Hayden Lau, Krishna Sri Ipsit Mantri, Ferdinand Mom, Atsushi Saito, Xiangru Tang, Bolun Wang, Johan S. Wind, Stansilaw Wozniak, Ruichong Zhang, Zhenyuan Zhang, Qihang Zhao, Peng Zhou, Jian Zhu, and Rui-Jie Zhu, \"RWKV: Reinventing RNNs for the Transformer Era,\" arXiv<\/em>, 2023.
      [4] Kunihiko Fukushima, \"Neocognitron: A self-organizing neural network model for a mechanism of pattern recognition unaffected by shift in position,\" Biological Cybernetics<\/em>, 1980.
      [5] Tom Michael Mitchell, \"The Need for Biases in Learning Generalizations,\" 2007.
      [6] Andrew Jaegle, Felix Gimeno, Andrew Brock, Andrew Zisserman, Oriol Vinyals and Joao Carreira. \"Perceiver: General Perception with Iterative Attention,\" in Proc. ICML<\/em>, 2021.
      [7] Weihao Yu, Mi Luo, Pan Zhou, Chenyang Si, Yichen Zhou, Xinchao Wang, Jiashi Feng, and Shuicheng Yan. \"MetaFormer is Actually What You Need for Vision,\" in Proc. CVPR<\/em>, 2021.
      [8] Shuangfei Zhai, Walter Talbott, Nitish Srivastava, Chen Huang, Hanlin Goh, Ruixiang Zhang, and Josh Susskind, \"An Attention Free Transformer,\" arXiv<\/em>, 2021.<\/p>\n","protected":false},"excerpt":{"rendered":"

      \u306f\u3058\u3081\u306b \u672c\u8a18\u4e8b\u3067\u306f\u3001Attention Free Transformer[1]\u3068\u547c\u3070\u308c\u308b\u30e2\u30c7\u30eb\u306e\u30a2\u30fc\u30ad\u30c6\u30af\u30c1\u30e3\u306b\u3064\u3044\u3066\u89e3\u8aac\u3057\u307e\u3059\u3002 \u306f\u3058\u3081\u306b\u5927\u67a0\u3092\u8aac\u660e\u3057\u307e\u3059\u3002\u8fd1\u5e74\u3067\u306fTransformer[2]\u3068\u547c … <\/p>\n","protected":false},"author":1,"featured_media":8960,"comment_status":"closed","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_vk_print_noindex":"","sitemap_hide":"","_veu_custom_css":"","veu_display_promotion_alert":"common","vkexunit_cta_each_option":"","_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[170,171,106,174,206],"tags":[],"class_list":["post-8227","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-blog","category-learn","category-ml","category-dl","category-206"],"veu_head_title_object":{"title":"","add_site_title":""},"jetpack_featured_media_url":"https:\/\/developers.agirobots.com\/jp\/wp-content\/uploads\/2023\/08\/IMG_2695.png","jetpack-related-posts":[],"jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/posts\/8227","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/comments?post=8227"}],"version-history":[{"count":61,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/posts\/8227\/revisions"}],"predecessor-version":[{"id":9393,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/posts\/8227\/revisions\/9393"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/media\/8960"}],"wp:attachment":[{"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/media?parent=8227"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/categories?post=8227"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/developers.agirobots.com\/jp\/wp-json\/wp\/v2\/tags?post=8227"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}