U
    E‹ÑhÈé ã                   @   sD&  d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z	d dl
mZ d dl	mZmZmZ d dlmZmZmZmZ d dlmZ d dlmZmZ d dl
mZmZmZmZmZmZm Z m!Z! d d	l"m#Z#m$Z$m%Z%m&Z&m'Z' d d
l(m)Z)m*Z* d dl+m,Z- e	j.j/Z/e	j0 1ddd¡Z2dd„ Z3edœdd„Z4dd„ Z5dd„ Z6e3e/j7e/j8gƒe'ƒ ddde	j9ddfdd„ƒƒZ:e3e/j;j<e/j;j=gƒe'ƒ dd„ ƒƒZ>e3e/j?j<e/j?j=gƒe'ƒ ddœdd „ƒƒZ?e3e/j@ƒe'ƒ d!d"„ ƒƒZ@e3e/jAj<e/jAj=e/jBj<e/jBj=gƒe'd#d$ƒd%d&„ ƒƒZCe3e/jDj<e/jDj=gƒe'ƒ d'd(„ ƒƒZDd)d*„ ZEe3e/jFj<e/jFj=gƒe'ƒ d+d,„ ƒƒZGe3e/jHj<e/jHj=gƒe'ƒ d-d.„ ƒƒZIe3e/jJjKƒdd/œd0d1„ƒZLe3e/jJj<ƒe	jMdddd2œd3d4„ƒZNe3e/jOj<e/jOj=gƒe'ƒ e	jMdddd2œd5d6„ƒƒZPe3e/jOjQe/jOjRgƒe'ƒ e	jMdddd2œd7d8„ƒƒZSe3e/jTj<e/jTj=gƒe'ƒ ddddd2œd9d:„ƒƒZUe3e/jVj<e/jVj=gƒe'ƒ d;d<„ ƒƒZWe3e/jXj<ƒ�d˜d=d>„ƒZYd?d@„ ZZe3e/j[j<ƒdAdB„ ƒZ\e3e/j]ƒ�d™eeeee ee^ ee	j_ dCœdDdE„ƒZ`e3e/jaƒ�dšeeeee	j_ dFœdGdH„ƒZbe3e/jcƒdIdIddJœeeeeee	j_ dKœdLdM„ƒZde3e/jeƒ�d›e	je	jee ee ee	j_ efdNœdOdP„ƒZge3e/jhj<ƒdQdRœeeiee	je^efedSœdTdU„ƒZje3e/jkj<ƒdQdRœeeiee	je^efedSœdVdW„ƒZle'ƒ e3e/jmj<ƒdXdY„ ƒƒZne3e/joj<ƒdddd dddZœee^ee ee ee eiefed[œd\d]„ƒZpe3e/jqj<e/jqjrgƒe'ƒ d^d_„ ƒƒZse3e/jqjtƒ�dœd`da„ƒZue3e/jvj<e/jvjrgƒe'ƒ dbdc„ ƒƒZwe3e/jvjtƒ�d�ddde„ƒZxe3e/jyj<ƒdfdg„ ƒZze3e/jyj=ƒdhdi„ ƒZ{e3e/j|j<ƒdjdk„ ƒZ}e3e/j|j~ƒdldm„ ƒZe3e/j€j<ƒdndo„ ƒZ�e3e/j‚j<ƒddddddpœdqdr„ƒZƒe3e/j„j<ƒ�dždsdt„ƒZ„e3e/j…j<ƒ�dŸdudv„ƒZ†e3e/j‡j<ƒ�d dwdx„ƒZ‡e3e/jˆj<ƒdydz„ ƒZ‰e3e/jŠj~ƒd{d|„ ƒZ‹ee^d}œd~d„ZŒeee^d€œd�d‚„Z��d¡ee^efdƒœd„d…„ZŽ�d¢ee^e^d‡œdˆd‰„Z�eeefe^dŠœd‹dŒ„Z��d£e^eee^dŽœd�d�„Z‘e^d‘œd’d“„Z’e3e/j“j<e/j“j”gƒe'd”d•ƒ�d¤ee^efd—œd˜d™„ƒƒZ•e3e/j–j<e/j—j=gƒe'ƒ eedšœd›dœ„ƒƒZ˜e3e/j™gƒe'd”d•ƒed�œdždŸ„ƒƒZšeed œd¡d¢„Z›e3e/jœƒe'ƒ eeefed£œd¤d¥„ƒƒZœe3e/j�ƒe'ƒ �d¥eeefed£œd¦d§„ƒƒZ�e3e/jžƒe'ƒ �d¦eefed¨œd©dª„ƒƒZže3e/jŸƒe'ƒ �d§eefed¨œd«d¬„ƒƒZŸe3e/j j<ƒ�d¨eefefd­œd®d¯„ƒZ e3e/j¡j<e/j¡j=gƒe'ƒ eeed°œd±d²„ƒƒZ¡e3e/j¢j<ƒ�d©eefd³œd´dµ„ƒZ£e3e/j¤j<e/j¤j=gƒe'd¶d·d¸ƒddd¹œeefefeeeef dºœd»d¼„ƒƒZ¥e3e/j¦j<e/j¦j=gƒe'ƒ dd½œeeeefed¾œd¿dÀ„ƒƒZ§e3e/j¨j<e/j¨j=gƒe'dÁd–dÂƒdQdÃœeefeeeef dÄœdÅdÆ„ƒƒZ©e3e/jªj<e/jªj=gƒe'dÇd·d¸ƒdQddÈœeefefeeeef dÉœdÊdË„ƒƒZ«e3e/j¬j<e/j¬j=gƒe'ƒ dQddÌœeeeefefedÍœdÎdÏ„ƒƒZ­e3e/j®ƒe'dÁd–dÂƒ�dªeeefefeeeef dÐœdÑdÒ„ƒƒZ¯e^eefeff dÓœdÔdÕ„Z°e3e/j±j<e/j±j=gƒe'dÖd×ƒ�d«ee^eeef dÙœdÚdÛ„ƒƒZ²e3e/j³j<e/j³j´gƒe'dÜdÝdÇd·ƒeeeeeef dÞœdßdà„ƒƒZ³e3e/jµj<ƒ�d¬eefefee^ dáœdâdã„ƒZ¶eeeeei eei f däœdådæ„Z·eeee^ eeef dçœdèdé„Z¸eeefdêœdëdì„Z¹e3e/jºƒdQddddddíœeeefefee ee ee ee eeeeef dîœ	dïdð„ƒZºe3e/j»j<e/j»j=gƒdQdddñœeeefefefee edòœdódô„ƒZ¼e3e/j½ƒe'dõdöƒ�d­eeefefefeeef d÷œdødù„ƒƒZ¾e3e/j¿j<ƒdúdû„ ƒZÀe3e/jÁƒe'ƒ �d®eeeefefedüœdýdþ„ƒƒZÁdÿ�d „ ZÂ�d�d„ ZÃe3e/jÄƒe'ƒ �d�d„ ƒƒZÅe3e/jÆƒe'ƒ �d�d„ ƒƒZÇ�d�d„ ZÈe3e/jÉƒe'�d	ƒ�d
�d„ ƒƒZÊe3e/jËƒe'�d	ƒ�d�d„ ƒƒZÌ�d�d„ ZÍe3e/jÎƒe'ƒ �d�d„ ƒƒZÏe3e/jÐƒe'ƒ �d�d„ ƒƒZÑe3e/jÒj<e/jÒjÓe/jÔj<e/jÔjÓgƒe'�d	ƒ�d�d„ ƒƒZÕ�d�d„ ZÖe3e/j×ƒe'ƒ �d�d„ ƒƒZØe3e/jÙƒe'ƒ �d�d„ ƒƒZÚe3e/jÛj<e/jÛjÓe/jÜj<e/jÜjÓgƒe'�d	ƒ�d�d„ ƒƒZÝe3e/jÞƒe'ƒ �d¯eeße�dœ�d �d!„ƒƒZàe3e/jáƒe'ƒ eeeßee�d"œ�d#�d$„ƒƒZâe3e/jãj<e/jãj=gƒe'ƒ dIdI�d%œ�d&�d'„ƒƒZäe3e/jåj<e/jåj=gƒe'ƒ dd/œ�d(�d)„ƒƒZæe3e/jçjßƒ�d°�d+�d,„ƒZèe3e/jåjéƒ�d±�d-�d.„ƒZêe3e/jëj<ƒ�d²�d/�d0„ƒZìe3e/jíƒe'ƒ �d1�d2„ ƒƒZî�d3�d4„ Zï�d5�d6œ�d7�d8„Zð�d³e	je	jeeei eif eeei eif eeei eif efeieeeei eif  �d9œ�d:�d;„Zñ�d<�d=„ Zòe3e/jój<ƒe	je	je	jeei eei eei efeei ei�d>œ	�d?�d@„ƒZôe	jõjö�rÈe	j0 1�dAdd¡Z÷e3e	j.jøjùj<ƒ�dB�dC„ ƒZúe3e	j.jøjûj<ƒ�dD�dE„ ƒZüe	jõjý�r"e	j0 1�dFdd¡Zþe3e	j.jÿ�j ƒ�dG�dH„ ƒ�Ze	j0 1�dIdd¡�Ze3e	j.�j�jj<ƒ�dJ�dK„ ƒ�Ze3e	j.�j�jj<ƒe3e	j.�j�j�jƒ�dL�dM„ ƒƒ�Ze	j0 1�dNdd¡�Z	e3e	j.�j
�jƒ�d´�dR�dS„ƒ�Z�dT�dU„ �Ze3e/�jj<ƒ�dµ�dV�dW„ƒ�Z�dX�dY„ �Ze3e/�jj<ƒ�dZ�d[„ ƒ�Ze3e/�jƒe'ƒ �d¶�d\�d]„ƒƒ�Ze3e/�jƒe'�d	ƒ�d^�d_„ ƒƒ�Ze3e/�jj<ƒ�d`�da„ ƒ�Ze3e/�jj<ƒ�db�dc„ ƒ�Ze3e/�jj<ƒ�dd�de„ ƒ�Ze3e/�jƒe'�d	ƒ�df�dg„ ƒƒ�Zee^�dhœ�di�dj„�Ze3e/�j ƒe'�dkd$ƒ�dl�dm„ ƒƒ�Z!e3e/�j"ƒe'�d	ƒ�dn�do„ ƒƒ�Z#e3e/�j$ƒe'�dkd$ƒ�dp�dq„ ƒƒ�Z%e3e/�j&ƒe'�d	ƒ�dr�ds„ ƒƒ�Z'e3e/�j(jƒ�d·�dt�du„ƒ�Z)e3e/�j*j<e/�j*j=gƒe'ƒ �dv�dw„ ƒƒ�Z+e3e/�j,j<e/�j,j=gƒe'ƒ d�dxœeiei�dyœ�dz�d{„ƒƒ�Z,e3e/�j-je/�j.jgƒ�d|�d}„ ƒ�Z/e3e/�j0j<gƒ�d~�d„ ƒ�Z1e3e/�j2j<e/�j2j=gƒe'ƒ dIdI�d%œ�d€�d�„ƒƒ�Z3e3e/�j4j<gƒdd�d‚œ�dƒ�d„„ƒ�Z5e3e/�j6j<gƒdd�d‚œ�d…�d†„ƒ�Z7e3e/�j8gƒe'ƒ �d‡�dˆ„ ƒƒ�Z9e3e/�j:gƒ�d‰�dŠ„ ƒ�Z;e3e/�j<gƒ�d‹�dŒ„ ƒ�Z=e3e/�j>gƒ�d��dŽ„ ƒ�Z?e3e/�j@j<ƒ�d��d�„ ƒ�ZAe3e/�jBƒe'ƒ �d‘�d’„ ƒƒ�ZCe3e/�jDj<ƒ�d¸�d“�d”„ƒ�ZEe3e/�jFj<ƒ�d•�d–„ ƒ�ZG�d¹�d—�d˜„�ZHe3e/�jIj<e/�jIj=gƒe'ƒ �dºd�d™œ�dš�d›„ƒƒ�ZJe3e/�jKj<e/�jLj<gƒ�dœ�d�„ ƒ�ZMe3e/�jKjte/�jK�jNe/�jLjte/�jL�jNe/�jOj<e/�jO�jPgƒe'd#d$ƒ�d»�dž�dŸ„ƒƒ�ZQe3e/�jRj<ƒ�d �d¡„ ƒ�ZSe3e/�jTj<ƒ�d¢�d£„ ƒ�ZUe3e/�jVj<ƒ�d¤�d¥„ ƒ�ZWe3e/�jX�jYe/�jZ�jYe/�jXje/�jZje/�j[j<e/�j\j<e/�j]j<gƒ�d¦�d§„ ƒ�Z^e3e/�j_�jYe/�j`�jYe/�j_je/�j`jgƒ�d¼�d¨�d©„ƒ�Zae3e/�jbj<e/�jb�jcgƒ�dª�d«„ ƒ�Zd�d¬�d­„ �Zee3e/�jfje/�jf�jYgƒ�d®�d¯„ ƒ�Zge3e/�jhje/�jh�jYgƒ�d°�d±„ ƒ�Zie3e/�jjj<ƒ�d²�d³„ ƒ�Zke3e/�jlje/�jl�jYgƒ�d´�dµ„ ƒ�Zme3e/�jnje/�jn�jYgƒ�d¶�d·„ ƒ�Zoe3e/�jpj<ƒ�d¸�d¹„ ƒ�Zqe3e/�jrj<e/�jsj<gƒ�d½�dº�d»„ƒ�Zte3e/�ju�jYƒ�d¼�d½„ ƒ�Zve3e/�jwj<ƒ�d¾�d¿„ ƒ�Zxe3e/�jyƒ�dÀ�dÁ„ ƒ�Zze3e/�j{ƒe'ƒ �dÂ�dÃ„ ƒƒ�Z|e3e/�j}ƒ�dÄ�dÅ„ ƒ�Z~e3e/�jj<ƒ�d¾�dÆ�dÇ„ƒ�Z€e3e/�j�j<ƒ�dÈ�dÉ„ ƒ�Z‚�d¿�dÊ�dË„�Zƒe3e/�j„j<ƒ�dÌ�dÍ„ ƒ�Z…�dÎ�dÏ„ �Z†�dÐ�dÑ„ �Z‡�dÒ�dÓ„ �Zˆ�dÔ�dÕ„ �Z‰�dÀeeieieieieieieieieieieieieieieieieieieie^ef�dÖœ�d×�dØ„�ZŠ�dÙ�dÚ„ �Z‹eeeieieieieieieieieieieieieieieieie^�dÛœ�dÜ�dÝ„�ZŒ�dÞ�dß„ �Z�e3e/�jŽj<ƒ�dà�dá„ ƒ�Z�e3e/�j�j<ƒ�dÁ�dâ�dã„ƒ�Z‘e3e/�j’j<ƒ�dä�då„ ƒ�Z“e3e/�j”ƒe'ƒ �dæ�dç„ ƒƒ�Z•�dè�dé„ �Z–e3e/�j—ƒe'ƒ �dê�dë„ ƒƒ�Z˜e3e/�j™ƒe'�dkd$ƒ�dÂ�dì�dí„ƒƒ�Zše3e/�j›ƒe'�d	ƒ�dî�dï„ ƒƒ�Zœee�dðœ�dñ�dò„�Z�G �dó�dô„ �dôeƒ�Zžeeei�dõœ�dö�d÷„�ZŸe3e/�j j<ƒ�dø�dù„ ƒ�Z¡e3e/�j¢ƒe'ƒ �dú�dû„ ƒƒ�Z¢e3e/�j£ƒe'�d	�düƒ�dý�dþ„ ƒƒ�Z£e3e/�j¤j<gƒ�dÿ�d „ ƒ�Z¤e3e/�j¥j<ƒ�dÃ�d�d„ƒ�Z¥e3e/�j¦jiƒ�d�d„ ƒ�Z§e3e/�j¨j<ƒ�d�d„ ƒ�Z©e3e/�jªj<ƒ�dÄ�d�d„ƒ�Z«�dÅeieief�d	œ�d
�d„�Z¬�d�d„ �Z­�d�d„ �Z®e3e/�j¯j<ƒ�dÆ�d�d„ƒ�Z°�dÇ�d�d„�Z±�dÈ�d�d„�Z²�d�d„ �Z³�dÉ�d�d„�Z´�dÊ�d�d„�Zµe3e/�j¶j<ƒ�d�d„ ƒ�Z·e3e/�j¸ƒ�d�d„ ƒ�Z¹e3e/�jº�j»e/�jº�j¼e/�jº�j½e/�jº�j¾gƒe'ƒ �dË�d �d!„ƒƒ�Z¿e3e/�jÀ�j»e/�jÀ�j¼e/�jÀ�j½e/�jÀ�j¾gƒ�dÌ�d"�d#„ƒ�ZÁe3e/�jÂgƒ�dÍeeeeeeeeeieießefeeeeß �d$œ�d%�d&„ƒ�ZÃe3e/�jÄgƒ�dÎeeeeßefee eeß �d(œ�d)�d*„ƒ�ZÅe3e/�jÆgƒ�dÏeeeeeeeßefee eeß �d+œ
�d,�d-„ƒ�ZÇe3e/�jÈgƒ�dÐeeeeee eeeeeßeef efeeß �d.œ�d/�d0„ƒ�ZÉe3e/�jÊgƒ�dÑeeeeeeeeeieießefeeeeß eei eei �d1œ�d2�d3„ƒ�ZËe3e/�jÌgƒ�dÒeeeeee ee ee e	�jÍe	�jÍeeßeeeiefeeß eei ef�d4œ�d5�d6„ƒ�ZÎe3e/�jÏj<gƒ�dÓe	je	jee	j ee	j_ ee	j ee	j ee	j ef�d7œ�d8�d9„ƒ�ZÐe3e/�jÑ�jÒe/�jÑ�jÓgƒe'ƒ �dÔ�d:�d;„ƒƒ�ZÔe3e/�jÕ�jÒƒ�dÕ�d<�d=„ƒ�ZÖe3e/�j×j<e/�j×j=gƒe'ƒ �dÖdd/œ�d>�d?„ƒƒ�ZØ�d@�dA„ �ZÙ�dB�dC„ �ZÚe3e/�jÛj<e/�jÜj<gƒ�d×�dD�dE„ƒ�ZÛe3e/�jÝj<e/�jÞj<gƒ�dØ�dF�dG„ƒ�ZÝe3e/�jßj<e/�jàj<gƒ�dÙeeeeie	�jÍf  eeeie	�jÍf  eeß eeß �dHœ�dI�dJ„ƒ�Zße3e/�jáj<e/�jâj<gƒ�dÚ�dK�dL„ƒ�Záe3e/�jãj<e/�jã�jäe/�jã�jPe/�jã�jågƒ�dÛ�dM�dN„ƒ�Zæe3e/�jç�jäƒdd�dOœ�dP�dQ„ƒ�Zè�dR�dS„ �Zée3e/�jêj<ƒ�dÜ�dT�dU„ƒ�Zëe3e/�jìj<ƒ�dV�dW„ ƒ�Zìe3e/�jíj<ƒ�dX�dY„ ƒ�Zí�dZ�d[„ �Zî�d\�d]„ �Zïe3e/�jðj<e/�jñj<gƒ�dÝ�d^�d_„ƒ�Zòe3e/�jój<ƒ�dÞ�d`�da„ƒ�Zóe3e/�jôj<ƒ�dß�db�dc„ƒ�Zõe	�jö�Z÷�dd�de„ �Zøe3e/�jùj<ƒ�df�dg„ ƒ�Zùe3e/�júj<ƒ�dh�di„ ƒ�Zúe3e/�jûj<ƒ�dj�dk„ ƒ�Züe3e/�jýj<ƒ�dl�dm„ ƒ�Zýe3e/�jþje/�jþ�jÿgƒe'ƒ dd�dnœ�do�dp„ƒƒ�Z e3e/�jgƒe'ƒ �dà�dr�ds„ƒƒ�Ze3e/�jj<e/�jj<gƒ�dá�dt�du„ƒ�Ze3e/�jj<ƒ�dv�dw„ ƒ�Ze3e/�jj<e/�jj=gƒe'ƒ �dâ�dx�dy„ƒƒ�Ze3e	j.j/�jƒ�dz�d{„ ƒ�Ze3e	j.j/�j	ƒ�d|�d}„ ƒ�Z	e3e/�j
ƒe'ƒ dddd�d~œ�d�d€„ƒƒ�Z�d��d‚„ �Ze3e/�jƒ�dã�dƒ�d„„ƒ�Ze3e/�jƒ�dä�d…�d†„ƒ�Ze3e/�jƒe'ƒ dd�d‡œ�dˆ�d‰„ƒƒ�Ze3e/�jƒe'ƒ eiee�dŠœ�d‹�dŒ„ƒƒ�Ze3e/�jj<ƒ�d��dŽ„ ƒ�Ze3e/�jƒe�d�œ�d��d‘„ƒ�Z�d’�d“„ �Z�d”�d•„ �Z�ee/�jƒ �ee/�jƒ �ee/�jƒ �ee/�jƒ �ee/�jƒ �ee/�j ƒ �ee/�j!ƒ �ee/�j"ƒ �ee/�j#ƒ �ee/�j$ƒ �ee/�j%ƒ �ee/�j&ƒ �ee/�j'ƒ �ee/�j(ƒ �ee/�j)ƒ �ee/�j*ƒ �ee/�j+ƒ �ee/�j,ƒ �ee/�j-ƒ �ee/�j.ƒ �ee/�j/ƒ d dl(Z	d d�l0Z	d d�l1Z	�d–�d—„ �Z2�e2ƒ  dS (å  é    N)ÚEnum)ÚListÚOptionalÚSequenceÚTupleÚUnion)ÚSymBoolÚSymFloatÚTensor)Ú_add_op_to_registryÚ_convert_out_paramsÚglobal_decomposition_tableÚ
meta_table)Ú
OpOverload)Ú_prim_elementwise_metaÚ$ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND)Úcorresponding_complex_dtypeÚcorresponding_real_dtypeÚelementwise_dtypesÚELEMENTWISE_TYPE_PROMOTION_KINDÚIntLikeÚmake_contiguous_strides_forÚNumberÚ
TensorLike)Ú_maybe_convert_to_dtypeÚ_maybe_resize_outÚ_resize_output_checkÚ_safe_copy_outÚout_wrapper)Ú_broadcast_shapesÚ_maybe_broadcast)Ú_pytreeÚatenZIMPLÚMetac                    s   ‡ fdd„}|S )Nc                    s$   t ˆ ƒ‰ ‡ fdd„}t |ˆ¡ ˆ S )Nc                    s   t t| ˆ ƒ d S ©N)r   r   ©Úop©Úfn© úG/tmp/user/2202/pip-unpacked-wheel-bf9o3bgk/torch/_meta_registrations.pyÚregister0   s    z0register_meta.<locals>.wrapper.<locals>.register)r   ÚpytreeZ	tree_map_)r(   r+   r%   r'   r*   Úwrapper-   s    zregister_meta.<locals>.wrapperr)   )r&   r-   r)   r%   r*   Úregister_meta,   s    	r.   ©Útype_promotionc                    s>   t j|d| iŽ\}‰ ‡ fdd„|D ƒ}t|Ž }t|dtjiŽS )NÚtype_promotion_kindc                    s   g | ]}t |ˆ ƒ‘qS r)   )r   ©Ú.0Úx©Úresult_dtyper)   r*   Ú
<listcomp>B   s     z$elementwise_meta.<locals>.<listcomp>r0   )Úutilsr   r    r   r   ÚDEFAULT)r0   ÚargsÚ_r)   r5   r*   Úelementwise_meta9   s    ÿþ
ÿÿr<   c                 C   s(   t jt jt jt jt jt ji}| | | ¡S r$   )ÚtorchZ	complex32ÚhalfÚcfloatÚfloatÚcdoubleÚdoubleÚget)ÚdtypeZfrom_complexr)   r)   r*   ÚtoRealValueTypeM   s       ýrE   c                    s0   t tˆf|žŽ ƒ‰ t ˆ ˆk‡ ‡fdd„¡ d S )Nc                      s   dˆ› dˆ › �S )Nzoutput with shape z# doesn't match the broadcast shape r)   r)   ©Zbroadcasted_shapeÚ
self_shaper)   r*   Ú<lambda>Z   ó    z)check_inplace_broadcast.<locals>.<lambda>)Útupler   r=   Ú_check)rG   Z
args_shaper)   rF   r*   Úcheck_inplace_broadcastV   s
    þrL   Fc	           	         s  t ˆtjƒr$t ˆ ¡ dkdd„ ¡ t ˆtjƒrHt ˆ ¡ dkdd„ ¡ tdd„ ˆˆˆfD ƒƒršt t ¡ ¡‰ ˆd kr|ˆ ‰q¦t t 	ˆ¡‡ ‡fdd„¡ nˆp¤t ¡ ‰t ˆtj
ƒs¶t‚t t ˆtƒ‡‡‡fdd„¡ t ˆtƒsât‚t ˆdkd	d„ ¡ tjˆfˆ|d
||d�S )Nr   c                   S   s   dS ©Nz:linspace only supports 0-dimensional start and end tensorsr)   r)   r)   r)   r*   rH   n   rI   z(meta_linspace_logspace.<locals>.<lambda>c                   S   s   dS rM   r)   r)   r)   r)   r*   rH   s   rI   c                 s   s   | ]}t |tƒV  qd S r$   )Ú
isinstanceÚcomplex)r3   Úargr)   r)   r*   Ú	<genexpr>v   s     z)meta_linspace_logspace.<locals>.<genexpr>c                      s   dˆ › dˆ› �S )Nzlinspace(): inferred dtype z& can't be safely cast to passed dtype r)   r)   )Údefault_complex_dtyperD   r)   r*   rH      rI   c                      s*   dt ˆƒj› dt ˆ ƒj› dt ˆƒj› d�S )Nz4received an invalid combination of arguments - got (ú, ú))ÚtypeÚ__name__r)   )ÚendÚstartÚstepsr)   r*   rH   ˆ   s   ÿÿÿc                   S   s   dS )Nz$number of steps must be non-negativer)   r)   r)   r)   r*   rH   Œ   rI   Úmeta)rD   ÚlayoutÚdeviceÚ
pin_memoryÚrequires_grad)rN   r=   r
   rK   ÚdimÚanyr8   r   Zget_default_dtypeÚis_complex_dtyperD   ÚAssertionErrorÚ_check_typer   Úempty)	rX   rW   rY   ÚbaserD   r\   r[   r]   r^   r)   )rR   rD   rW   rX   rY   r*   Úmeta_linspace_logspace^   sH    
þ
þÿþþúrf   c                    sN   t  ˆ jt jk‡ fdd„¡ t  |  ¡ dko6ˆ  ¡ dk dd„ ¡ |  ˆ j¡S )Nc                      s   dˆ j › �S )Nz2take(): Expected a long tensor for index, but got ©rD   r)   ©Úindexr)   r*   rH   ž   rI   zmeta_take.<locals>.<lambda>r   c                   S   s   dS )Nz*take(): tried to take from an empty tensorr)   r)   r)   r)   r*   rH   £   rI   )r=   rK   rD   ÚlongÚ_check_indexÚnumelÚ	new_emptyÚshape)Úselfri   r)   rh   r*   Ú	meta_take˜   s    

þþrp   éÿÿÿÿ©r_   c                   sh   ˆj }ˆj }t ||kdd„ ¡ t ˆ ˆ ¡dko>ˆ ˆ ¡dk‡ ‡‡fdd„¡ tˆjˆjƒ}ˆ |¡S )Nc                   S   s   dS )Nz=linalg.cross: inputs must have the same number of dimensions.r)   r)   r)   r)   r*   rH   ¯   rI   zlinalg_cross.<locals>.<lambda>é   c                      s"   dˆ › dˆ  ˆ ¡› dˆ  ˆ ¡› �S )Nzlinalg.cross: inputs dimension z must have length 3. Got ú and ©Úsizer)   ©r_   Úotherro   r)   r*   rH   ³   s    )Úndimr=   rK   rv   r   rn   rm   )ro   rx   r_   Zx_dZy_dÚ	out_shaper)   rw   r*   Úlinalg_cross¨   s    þþr{   c                 C   s$   t | dƒ t| dƒ tj| tjd�S )Nzlinalg.matrix_exp©Úmemory_format)ÚsquareCheckInputsÚcheckFloatingOrComplexr=   Ú
empty_likeÚcontiguous_format©ro   r)   r)   r*   Úlinalg_matrix_exp¼   s    

rƒ   ÚvaluesÚindicesc                 C   sV   t j| j| j| jd�}t j| j| jt jd�}|  ¡ dkrN| jdkrNt|| jƒ ||fS )N©r\   rD   r   )	r=   rd   rn   r\   rD   Úint64rl   ry   Úmaybe_wrap_dim)ro   r_   r„   r…   r)   r)   r*   Ú	cummaxminÄ   s
    r‰   c                 C   s   t || jƒ t | ¡ ¡ S r$   )rˆ   ry   r=   r€   Ú
contiguous)ro   r_   r)   r)   r*   ÚlogcumsumexpÑ   s    r‹   c                    sÎ  |j }t|ƒ}|| }tt|ƒƒ}dd„ t|ƒD ƒ}	|D ]}
d|	|
< q8g g  }}|D ]"}
|	|
 sl| |
¡ qT| |
¡ qT|| }t|ƒ}| ¡ ‰ |d |… }|j‡ fdd„dd� |||d …  }| |¡}dgt|j|d … ƒ }| 	|¡}| 
d¡}||d< |}tt|ƒƒD ]}|||  ||d	 < �q|  	|¡} d
d„ t|ƒD ƒ}d	}|d	 }|dk�r�||  d¡ ||| < ||||  9 }|d	8 }�qTt||ƒD ] }|  d	||  ¡||| < �qš|  |||  ¡ ¡S )Nc                 S   s   g | ]}d ‘qS )Fr)   ©r3   r;   r)   r)   r*   r7   â   s     z_exec_fft.<locals>.<listcomp>Tc                    s   ˆ |  S r$   r)   ©r4   ©Zself_stridesr)   r*   rH   ò   rI   z_exec_fft.<locals>.<lambda>©ÚkeyÚreverserq   r   é   c                 S   s   g | ]}d ‘qS ©r   r)   rŒ   r)   r)   r*   r7     s     )ry   ÚlenÚlistÚrangeÚappendÚstrideÚsortÚpermutern   Zreshaperv   Ú
as_stridedÚstorage_offset)Úoutro   Ú	out_sizesr_   Úforwardry   Zsignal_ndimÚ
batch_dimsZdim_permuteZis_transformed_dimÚdÚleftÚrightZ	batch_endÚtmpÚinputZbatched_sizesÚ
batch_sizeZbatched_out_sizesÚiZout_stridesZbatch_numelr)   rŽ   r*   Ú	_exec_fftÚ   sJ    






r¨   c                    sb   | j jst‚| j}|  |¡}|s$|S |d d … }|  ¡ ‰ |j‡ fdd„dd� t|| |||ƒ}|S )Nc                    s   ˆ |  S r$   r)   r�   rŽ   r)   r*   rH     rI   zmeta_fft_c2c.<locals>.<lambda>Tr�   )rD   Ú
is_complexrb   rn   rm   r˜   r™   r¨   )ro   r_   ÚnormalizationrŸ   rž   ÚoutputZsorted_dimsr)   rŽ   r*   Úmeta_fft_c2c  s    
r¬   c                 C   sR   | j jst‚t|  ¡ ƒ}|r<|d }|| d d }|||< | j|t | j ¡d�S )Nrq   é   r’   rg   )rD   Úis_floating_pointrb   r•   rv   rm   r8   r   )ro   r_   rª   ZonesidedÚoutput_sizesZlast_dimZlast_dim_halfsizer)   r)   r*   Úmeta_fft_r2c#  s     
ÿr°   )Ú	generatorc                C   s   t |t | g¡ƒS r$   )r   r=   ÚSize)Únr±   r�   r)   r)   r*   Úmeta_randperm3  s    r´   ©rD   r[   r\   r]   c                C   s   t j| ||||d�S ©Nrµ   ©r=   rd   )r³   rD   r[   r\   r]   r)   r)   r*   Úmeta_randperm_default8  s        ÿr¸   c                C   s   t j|||||d�S r¶   r·   )Úhighrv   rD   r[   r\   r]   r)   r)   r*   Úmeta_randintA  s        ÿrº   c                C   s   t j|||||d�S r¶   r·   )Úlowr¹   rv   rD   r[   r\   r]   r)   r)   r*   Úmeta_randint_lowK  s        ÿr¼   c                C   s   t j| ||||d�S r¶   r·   )rv   rD   r[   r\   r]   r)   r)   r*   Úmeta_rand_default\  s        ÿr½   c                 C   s8   | j jst‚t|  ¡ ƒ}|||d < | j|t| j ƒd�S )Nrq   rg   )rD   r©   rb   r•   rv   rm   rE   )ro   r_   rª   Zlastdimr¯   r)   r)   r*   Úmeta_fft_c2rd  s    r¾   c                 C   sR   t  | ¡dkrtdƒ‚t|tƒrN| | |¡}|  ¡ | ¡ krNtj 	||  ¡ ¡ | S )Nr’   zQmore than one element of the written-to tensor refers to a single memory location)
r=   Z_debug_has_internal_overlapÚRuntimeErrorrN   r
   Útorv   r"   Zexpand_copyÚdefault)ro   ÚsrcZnon_blockingZintermediater)   r)   r*   Ú
meta_copy_m  s    ÿ
rÃ   c                 C   sX   t |  ¡ ƒ}t |  ¡ ƒ}||  ¡ kr(dn|| ||  }| |d¡ | ||¡ ||fS ©Nr’   )r•   rv   r˜   r_   Úinsert)Útensorr_   Zresult_sizesZresult_stridesÚ
new_strider)   r)   r*   ÚinferUnsqueezeGeometry€  s     rÈ   c                 C   s0   t ||  ¡ d ƒ}t| |ƒ\}}|  ||¡ | S rÄ   )rˆ   r_   rÈ   Úas_strided_)ro   r_   Zg_sizesZ	g_stridesr)   r)   r*   Úmeta_unsqueeze_‰  s    rÊ   )r¥   ÚweightÚ_metaÚbiasÚ_activation_optÚ	out_dtypec           	      C   sÊ   t | jƒ}|d k	r.| d¡| d¡ks.tdƒ‚| d¡|  d¡d ksJt‚| d¡|d< t| jƒdksntdƒ‚d|  d¡f}|d k	r¢| jtjkrš|tjks¢tdƒ‚| j	||d kr¶| jn|d� 
||¡}|S )	Nr   zoutput size mismatchr’   rq   r­   z*we can only handle the squashed input caseú9out_dtype is only supported for i8i8->i32 linear operatorrg   )r•   rn   rv   rb   r”   rD   r=   Úint8Úint32rm   r›   )	r¥   rË   rÌ   rÍ   rÎ   rÏ   r¯   Ztransposed_stridesr«   r)   r)   r*   Úmeta_sparse_structured_linear‘  s,    	

ÿÿþþ ýrÓ   )Úmat1Ú	mat1_metaÚmat2rÏ   c                 C   s¬   t | jƒdkst‚t |jƒdks$t‚t |jƒdks6t‚|  d¡| d¡d ksRt‚|  d¡| d¡g}|d k	rŒ|jtjkr„|tjksŒtdƒ‚|j||d kr |jn|d�}|S )Nr­   r’   r   rÐ   rg   ©	r”   rn   rb   rv   rD   r=   rÑ   rÒ   rm   )rÔ   rÕ   rÖ   rÏ   r¯   r«   r)   r)   r*   Úmeta_sparse_structured_mm³  s"    
ÿÿþþrØ   r’   )ÚalphaÚbetarÏ   )r¥   rÔ   rÕ   rÖ   rÏ   c          	      C   sÞ   t | jƒdkstdƒ‚t |jƒdks(t‚t |jƒdks:t‚t |jƒdksLt‚|  d¡| d¡kshtdƒ‚| d¡| d¡d ks„t‚| d¡| d¡g}|d k	r¾|jtjkr¶|tjks¾tdƒ‚|j||d krÒ|jn|d�}|S )Nr’   zEonly input broadcasted to columns of mat1 * mat2 product is supportedr­   r   rÐ   rg   r×   )	r¥   rÔ   rÕ   rÖ   rÙ   rÚ   rÏ   r¯   r«   r)   r)   r*   Úmeta_sparse_structured_addmmÌ  s4    ÿþÿþ
ÿÿþþrÛ   )Úcompressed_AÚdense_BrÍ   rÙ   rÏ   Útranspose_resultc                 C   sô   |j tjtjtjtjhks"tdƒ‚| j |j ks6tdƒ‚t|jƒdksLtdƒ‚| j tjk}|r`dnd}| 	d¡}| 	d¡}	|  
¡ d	 ||  }
|d k	r¦|
| 	d¡ks¦t‚|d k	rÎ|rÆ|tjtjtjhksÎtd
ƒ‚|rÚ|	|
fn|
|	f}|j||d�}|S )Nz2_cslt_sparse_mm only supports fp16, bf16, and int8zinputs must have the same dtyper­   z'_cslt_sparse_mm only supports 2d inputsé
   é	   r   r’   é   z?out_dtype is only supported for i8i8->fp16, bf16, or i32 matmulrg   )rD   r=   Úfloat32Úfloat16Úbfloat16rÑ   rb   r”   rn   rv   rl   rÒ   rm   )rÜ   rÝ   rÍ   rÙ   rÏ   rÞ   Zis_int8_input_typeZcompression_factorÚkr³   ÚmÚoutput_shapeÚresultr)   r)   r*   Úmeta__cslt_sparse_mmï  s8    	üû

ýüré   T)Úinclude_self)ro   r_   ri   ÚsourceÚreducerê   Úreturnc                C   s   t j| t jd�S ©Nr|   )r=   r€   r�   ©ro   r_   ri   rë   rì   rê   r)   r)   r*   Úmeta_index_reduce  s    
rð   c                C   s   | S r$   r)   rï   r)   r)   r*   Úmeta_index_reduce_!  s    
rñ   c                 C   s.   t |  ¡ ƒ}|  ¡ dkr$| ¡ ||< |  |¡S )Nr   )r•   rv   r_   rl   rm   )ro   r_   ri   Úresult_sizer)   r)   r*   Úmeta_index_select/  s    ró   )Úlengthsr…   ÚoffsetsÚaxisÚunsafeÚinitial)Údatarì   rô   r…   rõ   rö   r÷   rí   c          
         sj   |d k	rt dƒ‚‡ ‡fdd„}|d k	r0||jƒS |d k	r^|jd d… |jd d f }	||	ƒS tdƒ‚d S )Nz?segment_reduce(): indices based reduction is not supported yet.c                    s(   t j| ˆjˆ d d …  ˆjdt jd�S )Nr’   rZ   ©rD   r\   r}   )r=   rd   rn   rD   r�   )Úlengths_shape©rö   rù   r)   r*   Úsegment_reduce_lengths_tensorI  s    üz:meta_segment_reduce.<locals>.segment_reduce_lengths_tensorrq   r’   z<segment_reduce(): Either lengths or offsets must be defined.)ÚNotImplementedErrorrn   r¿   )
rù   rì   rô   r…   rõ   rö   r÷   rø   rý   rû   r)   rü   r*   Úmeta_segment_reduce8  s    ÿ
rÿ   c                 C   s
   |   d¡S ©Nr)   ©rm   r‚   r)   r)   r*   Úmeta_max\  s    r  c                 C   s6   t  | j|f¡}t| ||ƒ}|  |¡| j|tjd�fS ©Nrg   ©r8   Úreduction_dimsrn   Ú_compute_reduction_shaperm   r=   rj   ©ro   r_   Úkeepdimrç   r)   r)   r*   Úmeta_max_dimb  s
    þr	  c                 C   s
   |   d¡S r   r  r‚   r)   r)   r*   Úmeta_minl  s    r
  c                 C   s6   t  | j|f¡}t| ||ƒ}|  |¡| j|tjd�fS r  r  r  r)   r)   r*   Úmeta_min_dimr  s
    þr  c                 C   s4   |   ¡ rt| jƒ}nt| tjd�\}}tj| |d�S )N©r1   rg   )r©   r   rD   r   r   ÚINT_TO_FLOATr=   r€   )ro   r6   r;   r)   r)   r*   Ú
meta_angle|  s    þ
r  c                 C   s$   t  ||  ¡ | j¡ | t  | ¡¡S r$   )r=   Z_resize_output_rv   r\   Úcopy_Úangle)ro   r�   r)   r)   r*   Úmeta_angle_outˆ  s    r  c                 C   s   d S r$   r)   )Úvalr)   r)   r*   Úassert_asyncŽ  s    r  c                 C   s   d S r$   r)   )r  Ú
assert_msgr)   r)   r*   Úassert_async_meta“  s    r  c                 C   s   d S r$   r)   )Úsr)   r)   r*   Ú
print_meta˜  s    r  ©rD   r[   r\   r]   r}   c                 C   s   t jddd�S )Nr   rZ   ©r\   r·   r  r)   r)   r*   Úmake_dep_token�  s    	r  c                 C   s4   ddl m} t| ttfƒr"tdƒ‚|| ||d� d S )Nr   )Úconstrain_rangeú'Constraining SymFloat or Symbool is nyi©ÚminÚmax)Ú%torch.fx.experimental.symbolic_shapesr  rN   r	   r   Ú
ValueError)rv   r  r  r  r)   r)   r*   Úsym_constrain_range©  s    r"  c                 C   s   t j| ||d� |S ©Nr  )r"   r"  ©rv   r  r  Ú	dep_tokenr)   r)   r*   Úfunctional_sym_constrain_range³  s    r&  c                 C   s4   ddl m} t| ttfƒr"tdƒ‚|| ||d� d S )Nr   )Ú_constrain_range_for_sizer  r  )r   r'  rN   r	   r   r!  )rv   r  r  r'  r)   r)   r*   Úsym_constrain_range_for_size¹  s    r(  c                 C   s   t j| ||d� |S r#  )r"   r(  r$  r)   r)   r*   Ú'functional_sym_constrain_range_for_sizeÃ  s    r)  c                 C   s   |S r$   r)   )r  r  r%  r)   r)   r*   Úfunctional_assert_async_metaÉ  s    r*  ©ro   Úf_namec                 C   sX   |   ¡ dkst|› d�ƒ‚|  d¡|  d¡ksTt|› d|  d¡› d|  d¡› d�ƒ‚d S )Nr­   z3: The input tensor must have at least 2 dimensions.rq   éþÿÿÿz5: A must be batches of square matrices, but they are ú by ú	 matrices)r_   rb   rv   r+  r)   r)   r*   r~   Ï  s    
ÿþÿ þr~   ©ro   ÚAÚnamec                    sŒ   t  ˆjˆ jk‡ ‡fdd„¡ t  ˆjˆ jk‡ ‡fdd„¡ t  ˆ  d¡ˆ  d¡k‡ fdd„¡ t  ˆ  d¡ˆ d¡k‡ ‡‡fdd„¡ d S )Nc                      s   dˆj › dˆ j › d�S )Nz:Expected b and A to be on the same device, but found b on z
 and A on ú	 instead.r  r)   ©r1  ro   r)   r*   rH   â  s    z(linearSolveCheckInputs.<locals>.<lambda>c                      s   dˆj › dˆ j › d�S )Nz=Expected b and A to have the same dtype, but found b of type z and A of type r3  rg   r)   r4  r)   r*   rH   ê  s    rq   r-  c                      s   dˆ   d¡› dˆ   d¡› d�S )Nz3A must be batches of square matrices, but they are r-  r.  rq   r/  ru   r)   ©r1  r)   r*   rH   ò  s    c                      s:   dˆ› dˆ   d¡› dˆ   d¡› dˆ  d¡› dˆ  d¡› �
S )NzIncompatible matrix sizes for z: each A matrix is rq   r.  z but each b matrix is r-  ru   r)   ©r1  r2  ro   r)   r*   rH   ú  s    )r=   rK   r\   rD   rv   r0  r)   r6  r*   ÚlinearSolveCheckInputsÛ  s     
þ
þ
þþr7  ©Útr,  Úallow_low_precision_dtypesc                    sZ   | j ‰ t |  ¡ p|  ¡ ‡ ‡fdd„¡ |sVt ˆ tjtjtjtjfk‡ ‡fdd„¡ d S )Nc                      s   ˆ› dˆ › �S )Nz<: Expected a floating point or complex tensor as input. Got r)   r)   ©rD   r,  r)   r*   rH   	  rI   z(checkFloatingOrComplex.<locals>.<lambda>c                      s   ˆ› dˆ › �S )Nz*: Low precision dtypes not supported. Got r)   r)   r;  r)   r*   rH     rI   )	rD   r=   rK   r®   r©   r@   rB   r?   rA   r8  r)   r;  r*   r     s    þþr   r1  ©r1  r,  Úarg_namec                    s"   t  |  ¡ dk‡ ‡fdd„¡ d S )Nr­   c                      s   ˆ› dˆ › d�S )Nz: The input tensor z! must have at least 2 dimensions.r)   r)   ©r=  r,  r)   r*   rH     rI   zcheckIsMatrix.<locals>.<lambda>)r=   rK   r_   r<  r)   r>  r*   ÚcheckIsMatrix  s    
þr?  ©r1  ÚBr¢   r,  c                    sZ   t ˆ ˆƒ tˆˆƒ t ˆr0ˆ  d¡ˆ d¡knˆ  d¡ˆ d¡k‡ ‡‡‡fdd„¡ d S )Nr-  rq   c                      sH   ˆ› dˆrdnd› dˆ   d¡› dˆ   d¡› dˆ  d¡› dˆ  d¡› d	�S )
Nz2: Incompatible shapes of A and B for the equation zAX = BzXA = Bz (r-  r4   rq   rt   rT   ru   r)   ©r1  rA  r,  r¢   r)   r*   rH   $  s    z#checkInputsSolver.<locals>.<lambda>)r~   r?  r=   rK   rv   r@  r)   rB  r*   ÚcheckInputsSolver  s    

*þrC  rè   ©Úfn_namerè   r¥   Úresult_namec                    s&   t  ˆjˆjk‡ ‡‡‡fdd„¡ d S )Nc                	      s$   ˆ › dˆ› dˆ› dˆj › dˆj › �	S )Nz: Expected z5 and input tensors to be on the same device, but got z on z and input on r  r)   ©rE  r¥   rè   rF  r)   r*   rH   1  s    z!checkSameDevice.<locals>.<lambda>)r=   rK   r\   rD  r)   rG  r*   ÚcheckSameDevice,  s    
þrH  ©ÚUPLOc                    s8   ˆ   ¡ }t tˆ ƒdko&|dkp&|dk‡ fdd„¡ d S )Nr’   ÚUÚLc                      s
   dˆ › �S )Nz1Expected UPLO argument to be 'L' or 'U', but got r)   r)   rI  r)   r*   rH   <  rI   zcheckUplo.<locals>.<lambda>)Úupperr=   rK   r”   )rJ  ZUPLO_uppercaser)   rI  r*   Ú	checkUplo8  s
    
þrN  ÚeigenvaluesZeigenvectorsrL  )r1  rJ  Ú	compute_vc                 C   sp   t | dƒ t|ƒ t| jƒ}|r@|  |¡}| |t|dd�¡ n|  dg¡}| ¡  | j|t| j	ƒd�}||fS )Nzlinalg.eighF©Z	row_majorr   rg   )
r~   rN  r•   rn   rm   rÉ   r   ÚpoprE   rD   )r1  rJ  rP  rn   ZvecsÚvalsr)   r)   r*   Úmeta__linalg_eigh@  s    


rT  )r¥   rí   c                 C   s@   t | dƒ t | j¡r| jn
t | j¡}| j| jd d… |d�S )Nzlinalg.eigvalsrq   rg   ©r~   r8   ra   rD   r   rm   rn   )r¥   Úcomplex_dtyper)   r)   r*   Úmeta__linalg_eigvalsW  s    

ÿ
ýrW  ©r¥   c                 C   sX   t | dƒ t | j¡r| jn
t | j¡}| j| jd d… |d�}| j| j|d�}||fS )Nz
linalg.eigrq   rg   rU  )r¥   rV  r„   Zvectorsr)   r)   r*   Úmeta_linalg_eigc  s    

ÿ
ýrY  )rÂ   rí   c                 C   s   | j jtjd� dd¡S )Nr|   r-  rq   )ZmTÚcloner=   r�   Ú	transpose)rÂ   r)   r)   r*   ÚcloneBatchedColumnMajorq  s    r\  )ro   r1  rM  rí   c                 C   s   t | ƒS r$   )r\  )ro   r1  rM  r)   r)   r*   Ú_cholesky_solve_helperu  s    r]  c                    sP   t  ˆjdk‡fdd„¡ t  ˆ jdk‡ fdd„¡ tˆˆ dƒ\}}t|||ƒS )Nr­   c                      s   dˆ j › d�S )Nz-b should have at least 2 dimensions, but has ú dimensions instead©ry   r)   r‚   r)   r*   rH   €  rI   z cholesky_solve.<locals>.<lambda>c                      s   dˆ j › d�S )Nz-u should have at least 2 dimensions, but has r^  r_  r)   r5  r)   r*   rH   „  rI   Úcholesky_solve)r=   rK   ry   Ú!_linalg_broadcast_batch_dims_namer]  )ro   r1  rM  Zself_broadcastedZA_broadcastedr)   r4  r*   r`  {  s    
þ
þ  ÿr`  )ro   rM  rí   c                 C   s.   |   ¡ dkrtj| tjd�S t| dƒ t| ƒS )Nr   r|   Úcholesky)rl   r=   r€   Úlegacy_contiguous_formatr~   r\  ©ro   rM  r)   r)   r*   rb  Œ  s    
rb  c                 C   s   t | dƒ t| ƒS )NÚcholesky_inverse)r~   r\  rd  r)   r)   r*   re  •  s    
re  )r1  rM  Úcheck_errorsc                 C   sf   t | dƒ t| dƒ | j}t|ƒ}t|dƒ}|  |¡}| ||¡ | j|d|d … tjd�}||fS )Nzlinalg.choleskyFr   r­   rg   )	r~   r   rn   r”   r   rm   rÉ   r=   rÒ   )r1  rM  rf  ZA_shapery   Z	L_stridesrL  Úinfosr)   r)   r*   Úlinalg_cholesky_ex�  s    



rh  )r¥   Útaurí   c                    s  t  ˆjdkdd„ ¡ t  ˆ d¡ˆ d¡kdd„ ¡ t  ˆ d¡ˆ d¡kdd„ ¡ t  ˆjˆj dk‡‡fd	d„¡ ˆjdkr¶ˆjd d… }ˆjd d… ‰ t  ˆ |k‡ fd
d„¡ t  ˆjˆjk‡‡fdd„¡ tdˆˆdƒ t jˆjtˆjdd�ˆjˆj	d�S )Nr­   c                   S   s   dS )NzHtorch.linalg.householder_product: input must have at least 2 dimensions.r)   r)   r)   r)   r*   rH   ¶  rI   z,linalg_householder_product.<locals>.<lambda>r-  rq   c                   S   s   dS )Nzbtorch.linalg.householder_product: input.shape[-2] must be greater than or equal to input.shape[-1]r)   r)   r)   r)   r*   rH   º  rI   c                   S   s   dS )Nz`torch.linalg.householder_product: input.shape[-1] must be greater than or equal to tau.shape[-1]r)   r)   r)   r)   r*   rH   ¾  rI   r’   c                      s   dˆj › dˆ j › �S )Nzptorch.linalg.householder_product: Expected tau to have one dimension less than input, but got tau.ndim equal to ú and input.ndim is equal to r_  r)   ©r¥   ri  r)   r*   rH   Ã  s    c                      s
   dˆ › �S )Nzltorch.linalg.householder_product: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r)   r)   ©Úactual_batch_tau_shaper)   r*   rH   Í  s    c                      s   dˆj › dˆ j › �S )Nz,torch.linalg.householder_product: tau dtype z does not match input dtype rg   r)   rk  r)   r*   rH   Õ  s    z torch.linalg.householder_productri  FrQ  ©rv   r˜   rD   r\   )
r=   rK   ry   rv   rn   rD   rH  Úempty_stridedr   r\   )r¥   ri  Zexpected_batch_tau_shaper)   )rm  r¥   ri  r*   Úlinalg_householder_product¯  sD    þþþþ

þ
þürp  )r1  rf  c                 C   s^   t | dƒ t| ddd� |  | j¡}| | jt| jdd�¡ | j| jd d… tjd�}||fS )Nzlinalg.inv_exF)r:  rQ  r-  rg   ©r~   r   rm   rn   rÉ   r   r=   rÒ   )r1  rf  rL  rg  r)   r)   r*   Úlinalg_inv_ex_metaå  s    
rr  ÚLDÚpivotsÚinfo)Ú	hermitianrf  )ro   rv  rf  rí   c                C   st   t | dƒ t| dƒ tj| jt| jdd�| j| jd�}| j| jd d… tj	d�}| j| jd d… tj	d�}|||fS )Nztorch.linalg.ldl_factor_exFrQ  rn  rq   rg   r-  )
r~   r   r=   ro  rn   r   rD   r\   rm   Úint)ro   rv  rf  rs  rt  ru  r)   r)   r*   Úlinalg_ldl_factor_ex_metañ  s    

ürx  )rv  )rs  rt  rA  rv  rí   c                   sÈ   t ˆdƒ tˆdƒ tˆ ˆdƒ t ˆ jdk‡ fdd„¡ ˆjd d… }t |ˆjk‡fdd„¡ t t ˆj	¡‡fdd„¡ t ˆj	ˆ j	k‡ ‡fdd„¡ t
ˆ ˆƒ\}}tj|t|d	d
�ˆ j	ˆ jd�S )Nztorch.linalg.ldl_solver­   c                      s   dˆ j › d�S )NzMtorch.linalg.ldl_solve: Expected B to have at least 2 dimensions, but it has r^  r_  r)   )rA  r)   r*   rH     s    z'linalg_ldl_solve_meta.<locals>.<lambda>rq   c                      s   dˆ j › d�S )Nzjtorch.linalg.ldl_solve: Expected LD.shape[:-1] and pivots.shape to be the same, but got pivots with shape ú instead©rn   r)   ©rt  r)   r*   rH     s    c                      s   dˆ j › �S )Nz<torch.linalg.ldl_solve: Expected pivots to be integers. Got rg   r)   r{  r)   r*   rH     rI   c                      s   dˆj › dˆ j › �S )Nz!torch.linalg.ldl_solve: LD dtype z does not match b dtype rg   r)   )rA  rs  r)   r*   rH   #  rI   FrQ  rn  )r~   r   r7  r=   rK   ry   rn   r8   Úis_integer_dtyperD   Ú_linalg_broadcast_batch_dimsro  r   r\   )rs  rt  rA  rv  Zexpected_pivots_shapeÚB_broadcast_sizer;   r)   )rA  rs  rt  r*   Úlinalg_ldl_solve_meta  s6    


þ
þ

þ
þ
ür  ÚPrK  )Úpivot)r1  r�  rí   c          	         s˜   t  ˆ jdk‡ fdd„¡ tˆ jƒ}|d }|d }t||ƒ}||d< |rVˆ  |¡}nˆ  dg¡}||d< ˆ  |¡}||d< ||d< ˆ  |¡}|||fS )Nr­   c                      s   dˆ j › d�S )Nz@linalg.lu: Expected tensor with 2 or more dimensions. Got size: ry  rz  r)   r5  r)   r*   rH   3  rI   z linalg_lu_meta.<locals>.<lambda>r-  rq   r   )r=   rK   ry   r•   rn   r  rm   )	r1  r�  Úsizesræ   r³   rå   r€  rL  rK  r)   r5  r*   Úlinalg_lu_meta.  s$    
þ



rƒ  ÚLU)r�  rf  )r1  r�  rf  rí   c          	         sš   t  ˆ jdk‡ fdd„¡ tˆ jƒ}|d }|d }t j|t|dd�ˆ jˆ jd�}| 	¡  t
||ƒ|d< ˆ j|t jd	�}| 	¡  ˆ j|t jd	�}|||fS )
Nr­   c                      s   dˆ j › d�S )NzFtorch.lu_factor: Expected tensor with 2 or more dimensions. Got size: ry  rz  r)   r5  r)   r*   rH   Q  rI   z*linalg_lu_factor_ex_meta.<locals>.<lambda>r-  rq   FrQ  rn  rg   )r=   rK   ry   r•   rn   ro  r   rD   r\   rR  r  rm   rw  )	r1  r�  rf  r‚  ræ   r³   r„  rt  ru  r)   r5  r*   Úlinalg_lu_factor_ex_metaJ  s&    
þ

ür…  )r¢   Úadjoint)r„  rt  rA  r¢   r†  rí   c                   sî   t ˆdƒ t ˆjˆ jk‡ ‡fdd„¡ t ˆjtjkdd„ ¡ tˆdƒ tˆˆ |dƒ t ˆ d¡ˆ d¡kdd„ ¡ t ˆjd d… ˆjk‡fdd„¡ t	ˆ ˆƒ\}}tj
|t|| d	�ˆ jˆ jd
�}| ¡ dkrê|sê| ¡ rê| ¡ }|S )Nztorch.linalg.lu_solvec                      s   dˆj › dˆ j › d�S )NzPlinalg.lu_solve: Expected LU and B to have the same dtype, but found LU of type ú and B of type ry  rg   r)   )rA  r„  r)   r*   rH   y  s    z&linalg_lu_solve_meta.<locals>.<lambda>c                   S   s   dS )NzElinalg.lu_solve: pivots should be a Tensor of scalar type torch.int32r)   r)   r)   r)   r*   rH   €  rI   zlinalg.lu_solverq   c                   S   s   dS )NzYlinalg.lu_solve: Number of pivots per batch should be same as the dimension of the matrixr)   r)   r)   r)   r*   rH   ˆ  rI   c                      s   dˆ j › d�S )Nzclinalg.lu_solve: Expected LU.shape[:-1] and pivots.shape to be the same, but got pivots with shape ry  rz  r)   r{  r)   r*   rH   Ž  s    rQ  rn  r   )r   r=   rK   rD   rw  r~   rC  rv   rn   r}  ro  r   r\   rl   r©   Zconj)r„  rt  rA  r¢   r†  r~  r;   rè   r)   )rA  r„  rt  r*   Úlinalg_lu_solve_metak  s<    

þ
þ
þ
þürˆ  )r„  rt  Úunpack_dataÚunpack_pivotsrí   c                    sÒ   t  ˆ jdk‡ fdd„¡ |r6t  |jt jkdd„ ¡ tˆ jƒ}|d }|d }t||ƒ}||d< |rrˆ  |¡}nˆ  dg¡}|r°||d< ˆ  |¡}	||d< ||d< ˆ  |¡}
nˆ  dg¡}	ˆ  dg¡}
||	|
fS )Nr­   c                      s   dˆ j › d�S )NzFtorch.lu_unpack: Expected tensor with 2 or more dimensions. Got size: ry  rz  r)   ©r„  r)   r*   rH   ®  rI   z lu_unpack_meta.<locals>.<lambda>c                   S   s   dS )Nz´torch.lu_unpack: LU_pivots is expected to be a contiguous tensor of torch.int32 dtype.
Note: this function is intended to be used with the output produced by torch.linalg.lu_factorr)   r)   r)   r)   r*   rH   ³  s    r-  rq   r   )	r=   rK   ry   rD   rÒ   r•   rn   r  rm   )r„  rt  r‰  rŠ  r‚  ræ   r³   rå   r€  rL  rK  r)   r‹  r*   Úlu_unpack_meta¤  s4    
þ
þ


rŒ  )Úmoderí   c                    sR   ˆ dkrd}d}n8ˆ dkr$d}d}n&ˆ dkr6d}d}nt  d‡ fdd„¡ ||fS )NÚreducedTZcompleteFÚrc                      s   dˆ › d�S )Nzqr received unrecognized mode 'z=' but expected one of 'reduced' (default), 'r', or 'complete'r)   r)   ©r�  r)   r*   rH   Û  s    z _parse_qr_mode.<locals>.<lambda>©r=   rK   )r�  Ú	compute_qrŽ  r)   r�  r*   Ú_parse_qr_modeÎ  s    
þr“  ÚQÚRrŽ  )r1  r�  rí   c                 C   sÌ   t | dƒ t| dƒ t|ƒ\}}| jd }| jd }t||ƒ}|r|t| jƒ}|rT|n||d< |  |¡}| |t|dd�¡ n|  dg¡}t| jƒ}	|sš|sž|n||	d< |  |	¡}
|
 |	t|	dd�¡ ||
fS )Nz	linalg.qrr-  rq   FrQ  r   )	r?  r   r“  rn   r  r•   rm   rÉ   r   )r1  r�  r’  Zreduced_moderæ   r³   rå   ZQ_shaper”  ZR_shaper•  r)   r)   r*   Úlinalg_qr_metaã  s"    








r–  ÚsignÚ	logabsdet)r1  rí   c                 C   sŠ   t | dƒ t| ddƒ | j}|  |d d… ¡}| j|d d… t| jƒd�}tj|t|dƒ| j| j	d�}| j|d d… tj
d�}||||fS )Nzlinalg.slogdetFr-  rg   rn  rq   )r~   r   rn   rm   rE   rD   r=   ro  r   r\   rÒ   )r1  rn   r—  r˜  r„  rt  r)   r)   r*   Ú_linalg_slogdet  s    
ür™  )r1  Úfull_matricesÚ
compute_uvÚdriverc                 C   sö   t | dƒ t| dƒ t| jd d… ƒ}| jd }| jd }t||ƒ}|rº|||rT|n|g }|  |¡}	|	 |t|dd�¡ ||r„|n||g }
|  |
¡}t| ƒdk}| |
t|
|d�¡ n|  dg¡}	|  dg¡}| j||g t	| j
ƒd�}|	||fS )	Nz
linalg.svdr-  rq   FrQ  Úcudar   rg   )r?  r   r•   rn   r  rm   rÉ   r   Údevice_hintrE   rD   )r1  rš  r›  rœ  r    ræ   r³   rå   ZU_shaperK  ZV_shapeÚVZis_cudaÚSr)   r)   r*   Ú_linalg_svd_meta  s$    






r¡  )Úarg1Úarg2rí   c                 C   sn   | j d d… }|j d d… }t||ƒ}t|ƒ}||  d¡|  d¡g7 }t|ƒ}|| d¡| d¡g7 }||fS )Nr-  rq   )rn   r   r•   rv   )r¢  r£  Zarg1_batch_sizesZarg2_batch_sizesÚexpand_batch_portionÚarg1_expand_sizeÚarg2_expand_sizer)   r)   r*   r}  <  s    
r}  )r¢  r£  r2  rí   c                 C   sV   |rt | ||ƒ t| |ƒ\}}|| jkr,| n|  |¡}||jkrD|n| |¡}||fS r$   )r7  r}  rn   Úexpand)r¢  r£  r2  r¥  r¦  Zarg1_broadcastedZarg2_broadcastedr)   r)   r*   ra  L  s    ÿÿra  )r¥   rx   rí   c                 C   s6   | j d d… }|jdkp0| jd |jko0|j |k}|S )Nrq   r’   )rn   ry   )r¥   rx   Zexpected_batched_rhs_shapeÚvector_caser)   r)   r*   Úlinalg_solve_is_vector_rhs^  s
    
ÿr©  )r¢   rf  rè   r„  rt  ru  )	r1  rA  r¢   rf  rè   r„  rt  ru  rí   c                   sr  t ˆ dƒ t ˆ jˆjk‡ ‡fdd„¡ tˆ ˆƒ}|r@ˆ d¡nˆ}	tˆ |	|dƒ t|	ˆ ƒ\}
}t |pl| dd„ ¡ |r†|
d d… n|
}tj|t	|| ƒˆjˆj
d�}ˆ j}ˆ j}tj|t	|dƒˆ jˆ j
d�}ˆ j|d d… tjd�}ˆ j|d d	… tjd�}||||f}||||f}td
d„ |D ƒƒ�rnt||ƒD ]6\}}t||jƒ | |j| ¡ ¡ t||dd� �q6|S )Nzlinalg.solvec                      s   dˆ j › dˆj › d�S )NzKlinalg.solve: Expected A and B to have the same dtype, but found A of type r‡  ry  rg   r)   ©r1  rA  r)   r*   rH   u  s    z"_linalg_solve_ex.<locals>.<lambda>rq   c                   S   s   dS )Nz•linalg.solve: Vector broadcasting of the left hand side is not supported for left=False. In this case linalg.solve is equivalent to B / A.squeeze(-1)r)   r)   r)   r)   r*   rH   €  s    rn  Frg   r-  c                 s   s   | ]}|d k	V  qd S r$   r)   r2   r)   r)   r*   rQ   ˜  s     z#_linalg_solve_ex.<locals>.<genexpr>)Ú	copy_fromÚcopy_toZexact_dtype)r   r=   rK   rD   r©  Ú	unsqueezerC  r}  ro  r   r\   rn   ry   rm   rÒ   ÚallÚzipr   rÉ   r˜   r   )r1  rA  r¢   rf  rè   r„  rt  ru  r¨  ÚB_ZB_broad_shaper;   Zresult_shapeZresult_rn   ry   ZLU_Zpivots_Zinfo_r�   Úresr�  Úor)   rª  r*   Ú_linalg_solve_exf  sL    

þ
þ
üür³  )r¢   Úunitriangularr�   )r1  rA  rM  r¢   r´  r�   rí   c          	      C   sš   |d kr|   dg¡}t|tƒs"t‚t| ||dƒ t|| d ƒ\}}| dd¡ ¡ oV| ¡ }|rjt	||j
ƒ}n,t||j
ƒr–| | dd¡j
¡ | dd¡ |S )Nr   zlinalg.solve_triangularr-  rq   )rm   rN   r   rb   rC  ra  r[  Úis_contiguousZis_conjr   rn   r   Zresize_Ú
transpose_)	r1  rA  rM  r¢   r´  r�   r°  ZA_Zavoid_copy_Ar)   r)   r*   Úlinalg_solve_triangular_meta¢  s    
r·  ÚsolutionÚcloned_coefficient)ro   r1  rM  r[  r´  rí   c           	         sà   t  ˆjdk‡fdd„¡ t  ˆ jdk‡ fdd„¡ tˆˆ dƒ ˆ jt jkr˜tˆˆ ƒ\}}t j|t|dd�ˆj	ˆj
d�}t j|t|dd�ˆ j	ˆ j
d�}n@ˆ jt jks°ˆ jt jkrÈt  ˆ¡}ˆ d	g¡}nt  dd
d„ ¡ ||fS )Nr­   c                      s   dˆ j › d�S )NzMtorch.triangular_solve: Expected b to have at least 2 dimensions, but it has r^  r_  r)   r‚   r)   r*   rH   Ç  s    z'triangular_solve_meta.<locals>.<lambda>c                      s   dˆ j › d�S )NzMtorch.triangular_solve: Expected A to have at least 2 dimensions, but it has r^  r_  r)   r5  r)   r*   rH   Î  s    Útriangular_solveFrQ  rn  r   c                   S   s   dS )Nz+triangular_solve: Got an unexpected layout.r)   r)   r)   r)   r*   rH   è  rI   )r=   rK   ry   r7  r[   Ústridedr}  ro  r   rD   r\   Ú
sparse_csrÚ
sparse_bsrr€   rm   )	ro   r1  rM  r[  r´  Zself_broadcast_sizeZA_broadcast_sizer¸  r¹  r)   r4  r*   Útriangular_solve_meta¼  s8    	
þ
þ
ü
ü
r¾  c                 C   sp   t | dƒ t| dƒ |  | jd d… ¡}|  | j¡}| | jt| jdd�¡ | j| jd d… tjd�}|||fS )Nz
linalg.detr-  FrQ  rq   rg   rq  )r1  Zdetr„  rt  r)   r)   r*   Ú_linalg_det_metaí  s    

r¿  )r¥   ri  rx   r¢   r[  rí   c                    sÀ  t  ˆjdkdd„ ¡ t  ˆjdkdd„ ¡ |r4dnd‰t  ˆjˆ ˆjd k‡fdd„¡ t  ˆjˆ ˆjd k‡fdd„¡ t  ˆjd ˆjd kd	d„ ¡ t  ˆjˆj d
k‡‡fdd„¡ t  ˆjˆjk‡‡fdd„¡ ˆjdk�rFˆjd d… }ˆjd d… ‰t  ˆ|k‡fdd„¡ ˆjd d… ‰ t  ˆ |k‡ fdd„¡ t  ˆjˆjk‡‡fdd„¡ t  ˆjˆjk‡‡fdd„¡ tdˆˆdƒ tdˆˆdƒ t jˆjtˆjdd�ˆjˆjd�S )Nr­   c                   S   s   dS )Nz3torch.ormqr: input must have at least 2 dimensions.r)   r)   r)   r)   r*   rH     rI   zormqr.<locals>.<lambda>c                   S   s   dS )Nz3torch.ormqr: other must have at least 2 dimensions.r)   r)   r)   r)   r*   rH     rI   r-  rq   c                      s   dˆ › d�S )Nútorch.ormqr: other.shape[z0] must be greater than or equal to tau.shape[-1]r)   r)   ©Úleft_size_conditionr)   r*   rH     rI   c                      s   dˆ › d�S )NrÀ  z"] must be equal to input.shape[-2]r)   r)   rÁ  r)   r*   rH     rI   c                   S   s   dS )NzHtorch.ormqr: tau.shape[-1] must be less than or equal to input.shape[-1]r)   r)   r)   r)   r*   rH     rI   r’   c                      s   dˆj › dˆ j › �S )Nz[torch.ormqr: Expected tau to have one dimension less than input, but got tau.ndim equal to rj  r_  r)   rk  r)   r*   rH     s    c                      s   dˆj › dˆ j › �S )Nzhtorch.ormqr: Expected other to have the same number of dimensions as input, but got other.ndim equal to rj  r_  r)   ©r¥   rx   r)   r*   rH   #  s    c                      s
   dˆ › �S )NzWtorch.ormqr: Expected batch dimensions of tau to be equal to input.shape[:-2], but got r)   r)   rl  r)   r*   rH   .  s    c                      s
   dˆ › �S )NzYtorch.ormqr: Expected batch dimensions of other to be equal to input.shape[:-2], but got r)   r)   )Úactual_batch_other_shaper)   r*   rH   7  s    c                      s   dˆ j › dˆj › �S )NzPtorch.ormqr: Expected input and tau to have the same dtype, but input has dtype z and tau has dtype rg   r)   rk  r)   r*   rH   ?  s    c                      s   dˆ j › dˆj › �S )NzRtorch.ormqr: Expected input and other to have the same dtype, but input has dtype z and other has dtype rg   r)   rÃ  r)   r*   rH   F  s    ztorch.ormqrri  rx   FrQ  rn  )	r=   rK   ry   rn   rD   rH  ro  r   r\   )r¥   ri  rx   r¢   r[  Zexpected_batch_shaper)   )rÄ  rm  r¥   rÂ  rx   ri  r*   Úormqrû  sr    	 ÿ ÿ
þ
þþþ
þ
þ
þ
þ
þürÅ  c                   s¦   t  tˆƒdˆ  k‡ ‡fdd„¡ ˆj}|ˆ d k}|}| }|rftd|ƒD ]}|o`ˆ |¡dk}qLn"td|ƒD ]}|o„ˆ |¡dk}qpt  |p’|‡ ‡fdd„¡ d S )Nr­   c                      s   ddˆ  › dt ˆƒ› �S )Nzpadding size is expected to be r­   z, but got: ©r”   r)   )r_   Úpaddingr)   r*   rH   Z  rI   z,_padding_check_valid_input.<locals>.<lambda>r’   r   c                      s    dˆ d › dˆ d › dˆj › �S )Nz	Expected r’   zD or r­   zcD (batch mode) tensor with possibly 0 batch size and other non-zero dimensions for input, but got: rz  r)   )r_   r¥   r)   r*   rH   o  s    )r=   rK   r”   ry   r–   rv   )r¥   rÇ  r_   Z	input_dimZis_batch_modeZvalid_batch_modeZvalid_non_batch_moder¡   r)   )r_   r¥   rÇ  r*   Ú_padding_check_valid_inputW  s"    þþrÈ  c                   sÖ   d}d‰ d}ˆj dkr0ˆ d¡}ˆ d7 ‰ |d7 }tˆ|dd� |\‰‰ˆ |¡}ˆ ˆ ¡‰ˆˆ ˆ ‰|r�t ˆˆk o|ˆˆk ‡ ‡‡‡fdd„¡ t ˆdk‡‡fdd„¡ ˆj dkrÂˆ |ˆf¡S ˆ ||ˆf¡S d S )	Nr   r’   rs   rr   c                      s   dˆ› dˆ› dˆ › dˆj › �S ©NzcArgument #4: Padding size should be less than the corresponding input dimension, but got: padding (rS   ú) at dimension ú
 of input rz  r)   ©Údim_wr¥   Úpad_lÚpad_rr)   r*   rH   ‹  s    z_pad1d_common.<locals>.<lambda>c                      s   dˆ › dˆ› �S )Nz
input (W: z%) is too small. Calculated output W: r)   r)   )Úinput_wÚoutput_wr)   r*   rH   “  rI   r­   )ry   rv   rÈ  r=   rK   rm   )r¥   rÇ  Úis_reflectionÚ	dim_planeÚnbatchÚnplaner)   )rÍ  r¥   rÐ  rÑ  rÎ  rÏ  r*   Ú_pad1d_commonv  s0    



þþ
rÖ  c                 C   s   t | |dd�S ©NT©rÒ  ©rÖ  ©r¥   rÇ  r)   r)   r*   Úmeta_reflection_pad1dœ  s    rÛ  c                 C   s   t | |dd�S ©NFrØ  rÙ  rÚ  r)   r)   r*   Úmeta_replication_pad1d¢  s    rÝ  c                   s¨   d‰ |s t  t|ƒdkdd„ ¡ ˆjdkr2ˆ d7 ‰ |\‰‰ˆ ˆ ¡}|ˆ ˆ ‰|rzt  ˆ|k ofˆ|k ‡ ‡‡‡fdd„¡ t  ˆˆ ˆ ¡k‡ ‡‡fdd„¡ ˆ ˆj¡S )Nr’   r­   c                   S   s   dS )Nz padding size is expected to be 2r)   r)   r)   r)   r*   rH   «  rI   z(_pad1d_backward_common.<locals>.<lambda>rs   c                      s   dˆ› dˆ› dˆ › dˆj › �S rÉ  rz  r)   rÌ  r)   r*   rH   ¸  s    c                      s   dˆ› dˆ  ˆ ¡› �S ©Nz(grad_output width unexpected. Expected: ú, Got: ru   r)   ©rÍ  Úgrad_outputrÑ  r)   r*   rH   À  rI   )r=   rK   r”   ry   rv   rm   rn   )rá  r¥   rÇ  rÒ  rÐ  r)   )rÍ  rá  r¥   rÑ  rÎ  rÏ  r*   Ú_pad1d_backward_common¨  s$    

þþrâ  Ú
grad_inputc                 C   s   t | ||dd�S r×  ©râ  ©rá  r¥   rÇ  r)   r)   r*   Úmeta_reflection_pad1d_backwardÆ  s    ræ  c                 C   s   t | ||dd�S rÜ  rä  rå  r)   r)   r*   Úmeta_replication_pad1d_backwardÌ  s    rç  c                   s8  d‰d‰ d}d}t ˆ|dd� ˆj}|dkrNˆ d¡}ˆd7 ‰ˆ d7 ‰ |d7 }|\‰‰	‰
‰ˆ |¡}ˆ ˆ ¡‰ˆ ˆ¡‰ˆˆ
 ˆ ‰ˆˆ ˆ	 ‰|ràt ˆˆk o¦ˆ	ˆk ‡‡‡‡	fdd„¡ t ˆ
ˆk oÌˆˆk ‡ ‡‡‡
fdd„¡ t ˆdkpòˆdk‡‡‡‡fd	d„¡ ˆjd
k�r"ˆ |ˆˆf¡S ˆ ||ˆˆf¡S d S )Nr­   r’   r   rr   é   c                      s   dˆ› dˆ› dˆ › dˆj › �S rÉ  rz  r)   rÌ  r)   r*   rH   ì  s    z_pad2d_common.<locals>.<lambda>c                      s   dˆ› dˆ› dˆ › dˆj › �S ©NzcArgument #6: Padding size should be less than the corresponding input dimension, but got: padding (rS   rÊ  rË  rz  r)   ©Údim_hr¥   Úpad_bÚpad_tr)   r*   rH   ó  s    c                      s   dˆ › dˆ› dˆ› dˆ› �S )Nz
input (H: ú W: z%) is too small. Calculated output H: r)   r)   )Úinput_hrÐ  Úoutput_hrÑ  r)   r*   rH   û  s    rs   ©rÈ  ry   rv   r=   rK   rm   )r¥   rÇ  rÒ  Z
dim_slicesrÔ  ry   rÕ  r)   )rë  rÍ  r¥   rï  rÐ  rð  rÑ  rì  rÎ  rÏ  rí  r*   Ú_pad2d_commonÒ  sB    



þþþrò  c                 C   s   t | |dd�S r×  ©rò  rÚ  r)   r)   r*   Úmeta_reflection_pad2d  s    rô  c                 C   s   t | |dd�S rÜ  ró  rÚ  r)   r)   r*   Úmeta_replication_pad2d  s    rõ  c                    sÎ   d‰d‰ d}d}|j }| ¡ dkrB|d }ˆd7 ‰ˆ d7 ‰ |d7 }|\}}}}	|| }
|ˆ  }|ˆ }|| |	 ‰|| | ‰t ˆˆ ˆ¡k‡‡‡fdd„¡ t ˆˆ ˆ ¡k‡ ‡‡fdd„¡ | |j ¡S )Nr­   r’   r   rè  c                      s   dˆ› dˆ  ˆ ¡› �S rÞ  ru   r)   rà  r)   r*   rH   3  rI   z%meta_pad2d_backward.<locals>.<lambda>c                      s   dˆ› dˆ  ˆ ¡› �S ©Nz)grad_output height unexpected. Expected: rß  ru   r)   ©rë  rá  rð  r)   r*   rH   7  rI   )rn   r_   r=   rK   rv   rm   )rá  ro   rÇ  rÓ  rÔ  rG   rÎ  rÏ  rí  rì  rÕ  rï  rÐ  r)   )rë  rÍ  rá  rð  rÑ  r*   Úmeta_pad2d_backward  s2    
þþrø  c             	      s’  d‰d‰d‰ d}t ˆ|dd� ˆjdk}|rVˆ d¡}ˆd7 ‰ˆd7 ‰ˆ d7 ‰ |d7 }|\‰‰‰‰
‰‰ˆ |¡}ˆ ˆ ¡‰ˆ ˆ¡‰ˆ ˆ¡‰ˆˆ ˆ ‰ˆˆ ˆ
 ‰ˆˆ ˆ ‰	|�r,t ˆˆk oÊˆˆk ‡‡‡‡fdd„¡ t ˆˆk oðˆ
ˆk ‡‡‡
‡fd	d„¡ t ˆˆk �oˆˆk ‡ ‡‡‡fd
d„¡ t ˆ	dk�pJˆdk�pJˆdk‡‡‡‡‡‡	fdd„¡ |�r|ˆ ||ˆˆˆ	f¡S ˆ |ˆˆˆ	f¡S d S )Nrs   r­   r’   r   rr   é   c                      s   dˆ› dˆ› dˆ › dˆj › �S rÉ  rz  r)   rÌ  r)   r*   rH   Y  s    z_pad3d_common.<locals>.<lambda>c                      s   dˆ› dˆ› dˆ › dˆj › �S ré  rz  r)   rê  r)   r*   rH   `  s    c                      s   dˆ› dˆ› dˆ › dˆj › �S )NzcArgument #8: Padding size should be less than the corresponding input dimension, but got: padding (rS   rÊ  rË  rz  r)   )Údim_dr¥   Úpad_bkÚpad_fr)   r*   rH   g  s    c                      s(   dˆ › dˆ› dˆ› dˆ› dˆ› dˆ› �S )Nz
input (D: ú H: rî  z%) is too small. Calculated output D: r)   r)   )Úinput_drï  rÐ  Úoutput_drð  rÑ  r)   r*   rH   o  s    rñ  )r¥   rÇ  rÒ  rÓ  Z
batch_moderÔ  rÕ  r)   )rú  rë  rÍ  r¥   rþ  rï  rÐ  rÿ  rð  rÑ  rì  rû  rü  rÎ  rÏ  rí  r*   Ú_pad3d_common<  sP    





þþþþr   c                 C   s   t | |dd�S r×  ©r   rÚ  r)   r)   r*   Úmeta_reflection_pad3d{  s    r  c                 C   s   t | |dd�S rÜ  r  rÚ  r)   r)   r*   Úmeta_replication_pad3d�  s    r  c                    s(  t  t|ƒdkdd„ ¡ |jdks&t‚ˆj|jks6t‚d‰d‰d‰ |jdkrdˆd7 ‰ˆd7 ‰ˆ d7 ‰ |\}}}}}}| ˆ ¡}	| ˆ¡}
| ˆ¡}|	| | ‰|
| | ‰|| | ‰t  ˆˆ ˆ¡k‡‡‡fdd„¡ t  ˆˆ ˆ¡k‡‡‡fd	d„¡ t  ˆˆ ˆ ¡k‡ ‡‡fd
d„¡ | |j¡S )Né   c                   S   s   dS )Nz padding size is expected to be 6r)   r)   r)   r)   r*   rH   ‘  rI   z%meta_pad3d_backward.<locals>.<lambda>rs   r­   r’   rù  c                      s   dˆ› dˆ  ˆ ¡› �S rÞ  ru   r)   rà  r)   r*   rH   ©  rI   c                      s   dˆ› dˆ  ˆ ¡› �S rö  ru   r)   r÷  r)   r*   rH   ­  rI   c                      s   dˆ› dˆ  ˆ ¡› �S )Nz(grad_output depth unexpected. Expected: rß  ru   r)   )rú  rá  rÿ  r)   r*   rH   ±  rI   )r=   rK   r”   ry   rb   rv   rm   rn   )rá  r¥   rÇ  rÎ  rÏ  rí  rì  rü  rû  rþ  rï  rÐ  r)   )rú  rë  rÍ  rá  rÿ  rð  rÑ  r*   Úmeta_pad3d_backward‡  s<    




þþþr  r­   )ro   Úprí   c                 C   sb   t  |  ¡ dd„ ¡ |  d¡}|dkr<|  dg¡jt jd�S |  ||d  d f¡jt jd�S d S )Nc                   S   s   dS )Nz(_pdist_forward requires contiguous inputr)   r)   r)   r)   r*   rH   »  rI   z%meta__pdist_forward.<locals>.<lambda>r   r’   r|   r­   )r=   rK   rµ  rv   rm   rÀ   rc  )ro   r  r³   r)   r)   r*   Úmeta__pdist_forward·  s     ÿ
ÿr  )Úgradro   r  Úpdistrí   c                 C   s8   t  | ¡ dd„ ¡ t  | ¡ dd„ ¡ t j|t jd�S )Nc                   S   s   dS )Nz._pdist_backward requires self to be contiguousr)   r)   r)   r)   r*   rH   Ê  rI   z&meta__pdist_backward.<locals>.<lambda>c                   S   s   dS )Nz/_pdist_backward requires pdist to be contiguousr)   r)   r)   r)   r*   rH   Í  rI   r|   )r=   rK   rµ  r€   rc  )r  ro   r  r	  r)   r)   r*   Úmeta__pdist_backwardÆ  s     ÿ ÿr
  )rÚ   rÙ   c          	         sæ   ˆ   d¡}ˆ   d¡}ˆ  d¡}ˆ |||f¡‰t ˆ  ¡ dkdd„ ¡ t ˆ ¡ dkdd„ ¡ t ˆjˆ j  kozˆjkn  ‡ ‡‡fdd„¡ ˆ j}ˆj‰|d ‰|d ‰t ˆd ˆkoÆˆd ˆk‡‡‡fd	d„¡ ˆ ˆ  ¡ ¡S )
Nr   r’   r­   rs   c                   S   s   dS ©Nzbatch1 must be a 3D tensorr)   r)   r)   r)   r*   rH   Ù  rI   zmeta_baddbmm.<locals>.<lambda>c                   S   s   dS ©Nzbatch2 must be a 3D tensorr)   r)   r)   r)   r*   rH   Ú  rI   c                      s   dˆj › dˆ j › dˆj › �S )Nz+Input dtypes must be the same, got: input: z
, batch1: z
, batch2: rg   r)   )Úbatch1Úbatch2ro   r)   r*   rH   Ý  rI   c                	      s&   dˆ› dˆ› dˆ d › dˆ d › d�	S ©Nz@Expected size for first two dimensions of batch2 tensor to be: [rS   z] but got: [r   r’   ú].r)   r)   ©Úbatch2_sizesÚbsÚcontraction_sizer)   r*   rH   å  s    )rv   r§  r=   rK   r_   rD   rn   rm   )	ro   r  r  rÚ   rÙ   Údim1Údim2Zdim3Úbatch1_sizesr)   )r  r  r  r  r  ro   r*   Úmeta_baddbmmÒ  s&    


þþr  c                C   s   t  | ¡ ¡ S r$   ©r=   r€   rŠ   )ro   r±   r)   r)   r*   Úmeta_bernoullií  s    r  ç      à?c                 C   s   | S r$   r)   ©ro   r  r±   r)   r)   r*   Úmeta_bernoulli_ô  s    r  c                 C   s   t  | ¡ ¡ S r$   r  r  r)   r)   r*   Úmeta_bernoulli_pù  s    r  c                 C   s6   t  |
|  ¡ k dd„ ¡ t j| t jd�}t  | ¡|fS )Nc                   S   s   dS )NzJError in fused_moving_avg_obs_fake_quant_cpu: ch_axis must be < self.dim()r)   r)   r)   r)   r*   rH     rI   z6meta__fused_moving_avg_obs_fq_helper.<locals>.<lambda>rg   )r=   rK   r_   r€   Úbool)ro   Zobserver_onZfake_quant_onZrunning_minZrunning_maxÚscaleZ
zero_pointZaveraging_constZ	quant_minZ	quant_maxZch_axisZper_row_fake_quantZsymmetric_quantÚmaskr)   r)   r*   Ú$meta__fused_moving_avg_obs_fq_helperÿ  s    
þr"  c                    sn   t  |  ¡ dkdd„ ¡ t  | ¡ dkdd„ ¡ | j\‰‰ |j\‰‰t  ˆ ˆk‡ ‡‡‡fdd„¡ |  ˆˆ¡S )Nr­   c                   S   s   dS )Nza must be 2Dr)   r)   r)   r)   r*   rH     rI   zmeta_mm.<locals>.<lambda>c                   S   s   dS )Nzb must be 2Dr)   r)   r)   r)   r*   rH     rI   c                	      s   dˆ› dˆ › dˆ› dˆ› d�	S )Nz/a and b must have same reduction dim, but got [rS   z] X [r  r)   r)   ©ZM1ZM2ÚNr€  r)   r*   rH      rI   )r=   rK   r_   rn   rm   ©ÚaÚbr)   r#  r*   Úmeta_mm  s    

þr(  c                    s0   |r"t ‡ ‡fdd„tˆjƒD ƒƒS t ˆjˆ ¡S )Nc                 3   s$   | ]}|ˆ krˆj | nd V  qdS )r’   Nrz  ©r3   r§   ©Údimsro   r)   r*   rQ   '  s     z+_compute_reduction_shape.<locals>.<genexpr>)rJ   r–   ry   r8   Úcompute_reduction_output_shapern   )ro   r+  r  r)   r*  r*   r  %  s    r  Ústr)rí   c                 C   s   t | tjjƒr| jjS dS d S )Nr�  )rN   r=   Z_subclassesZ
FakeTensorZfake_devicerU   )rÆ   r)   r)   r*   rž  0  s    rž  )Úinput_tensorrË   r˜   rÇ  ÚdilationÚis_transposedÚgroupsÚoutput_paddingc                 C   s  t t t t t t dœdd„}t t t t t t t dœdd„}	|jdd … }
| jdd … }|rb||jd  }n*|jd	 }|jd | | jd krŒtd
ƒ‚| jd	 |g}t|tƒr´|gt|ƒ }nt|ƒdkrÒ|d	 gt|ƒ }t|tƒrì|gt|ƒ }n t|ƒdk�r|d	 gt|ƒ }t|tƒ�r(|gt|ƒ }n t|ƒdk�rH|d	 gt|ƒ }d }|�r”t|tƒ�rn|gt|ƒ }n&t|ƒdk�r�|d	 gt|ƒ }n|}tt|ƒƒD ]h}|�rÜ| |	|| || || |
| || || ƒ¡ n*| ||| || || |
| || ƒ¡ �q |S )N)Úlnr  r¡   rå   r  rí   c                 S   s$   | d|  ||d   d | d S )a�  
        Formula to apply to calculate the length of some dimension of the output

        See: https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
        Returns:
            The output length
        r­   r’   r)   )r3  r  r¡   rå   r  r)   r)   r*   Ú_formulaA  s    z+calc_conv_nd_return_shape.<locals>._formula)r3  r  r¡   rå   r  r&   rí   c                 S   s(   | d | d|  ||d   | d S )aü  
        Formula to apply to calculate the length of some dimension of the output
        if transposed convolution is used.
        See: https://pytorch.org/docs/stable/generated/torch.nn.ConvTranspose2d.html

        Args:
            ln: length of the dimension
            p: padding in that dim
            d: dilation in that dim
            k: kernel size in that dim
            s: stride in that dim
            op: output padding in that dim

        Returns:
            The output length
        r’   r­   r)   )r3  r  r¡   rå   r  r&   r)   r)   r*   Ú_formula_transposedR  s    z6calc_conv_nd_return_shape.<locals>._formula_transposedr­   r’   r   zInvalid channel dimensions)rw  rn   r¿   rN   r   r”   r–   r—   )r.  rË   r˜   rÇ  r/  r0  r1  r2  r4  r5  Úkernel_sizer+  Zout_channelsZ	ret_shapeZoutput_padding_listr§   r)   r)   r*   Úcalc_conv_nd_return_shape7  sZ    



úÿ"ÿr7  c                 C   s   t j | ¡t jkS r$   ©r=   Ú_prims_commonÚsuggest_memory_formatÚchannels_last©Ztenr)   r)   r*   Úis_channels_lastœ  s    r=  )	r.  rË   rÍ   r˜   rÇ  r/  r0  r2  r1  c	              	      sf   ‡ ‡fdd„}	t ˆ ˆ||||||r&|nd ƒ}
d}d}ˆ  |¡dkrJd|
|< ˆ  |
¡}|j|	ƒ d�}|S )Nc                      s^   t ˆ ƒdkr$tˆ ƒstˆƒr2tjS ntˆ ƒr2tjS ˆ jtjd�rFtjS ˆ jtjd�rZtjS d S ©Nr�  r|   )rž  r=  r=   r;  rµ  r�   Úpreserve_formatr)   ©r.  rË   r)   r*   Úpick_memory_format¬  s    z%meta_conv.<locals>.pick_memory_formatr’   r   r|   )r7  rv   rm   rÀ   )r.  rË   rÍ   r˜   rÇ  r/  r0  r2  r1  rA  Ú	shape_outZinput_channels_dimZoutput_channels_dimr�   r)   r@  r*   Ú	meta_conv   s$    
ø
rC  Úmkldnnc
              	   C   sH   t | ||||d|g ƒ}
|  |
¡}tj}|  ¡ dkr8tj}|j|d�}|S )NFrù  r|   )r7  rm   r=   r;  r_   Úchannels_last_3drÀ   )r.  rË   rÍ   rÇ  r˜   r/  r1  ÚattrÚscalarsÚ	algorithmrB  r�   Zout_memory_formatr)   r)   r*   Úmeta_mkldnn_convolution_defaultÒ  s            ÿ
rI  c                 C   s    |   | jd d… |jd f˜¡S ©Nrq   r   ©rm   rn   )r.  rË   rÍ   rF  rG  rH  r)   r)   r*   Úmeta_linear_pointwise_defaulté  s    rL  Úmklc                 C   s    |   | jd d… |jd f˜¡S rJ  rK  )r.  Zpacked_weightZorig_weightrÍ   r¦   r)   r)   r*   Úmeta_mkl_linearô  s    ÿrN  Úonednnc              	   C   sJ   t | ||||	d|
d ƒ}|tjtjfks*t‚| j||d�}|jtjd�}|S )NFrg   r|   )r7  r=   râ   rä   rb   rm   rÀ   r;  )r4   Úx_scaleÚx_zpÚwÚw_scaleÚw_zprÍ   r˜   rÇ  r/  r1  Úoutput_scaleÚoutput_zero_pointÚoutput_dtyperF  rG  rH  rB  r�   r)   r)   r*   Úmeta_qconv2d_pointwise	  s    ø
rX  c                 C   s>   t | jƒ}|jd |d< |	tjtjfks,t‚| j||	d�}|S )Nr’   rq   rg   )r•   rn   r=   râ   rä   rb   rm   )r4   rP  rQ  rR  rS  rT  rÍ   rU  rV  rW  Zpost_op_nameZpost_op_argsZpost_op_algorithmrç   r�   r)   r)   r*   Úmeta_qlinear_pointwise'	  s
    
rY  Ú	quantizedr)   r“   ©r’   c                 C   sr   t | |||||ƒ\}}}|  ¡ dkr.|  d¡nd}	tj}
|  ¡ dkrP|||g}n|	|||g}tj|| j| j|
d�S ©Nrè  éüÿÿÿr’   rs   rú   )Ú#max_pool2d_checks_and_compute_shaper_   rv   r=   r;  rd   rD   r\   ©r¥   r6  r˜   rÇ  r/  Ú	ceil_modeÚnInputPlaneÚoutputHeightÚoutputWidthrÔ  r}   rv   r)   r)   r*   Úmeta_quantized_max_pool2dC	  s.         ÿüürd  c                    s4   t  ˆ ¡ ˆ koˆjˆ ˆk‡ ‡‡‡fdd„¡ d S )Nc                      s8   dˆ › dˆ› dˆ› d�dˆ  ¡ › dˆ› dˆjˆ › � S )NzExpected a tensor of dimension z and tensor.size[z] == rS   zbut got : dimension z] = ©r_   rn   r)   ©r_   Údim_sizerv   rÆ   r)   r*   rH   e	  s   ÿz check_dim_size.<locals>.<lambda>)r=   rK   r_   rn   )rÆ   r_   rg  rv   r)   rf  r*   Úcheck_dim_sizeb	  s    þrh  c                 C   sd  dd„ }|d|ƒ\}}	t  t|ƒdkdd„ ¡ t|ƒdkrF||	 }
}n.t|ƒdkrf|d |d  }
}n|d	|ƒ\}
}|d
|ƒ\}}t  |d kp”|dkdd„ ¡ |  ¡ dkr´|  d¡nd}|  d¡}|  d¡}|  d¡}t||||
d|ƒ}t||	||d|ƒ}t | ¡}t| ||	|
|||dd||||||ƒ |  ¡ dk�rB|||g}n||||g}t j	|| j
| j|d�S )Nc                    sD   t  t|ƒdk‡ fdd„¡ |d }t|ƒdkr4|n|d }||fS )N©r’   r­   c                      s   dˆ › d�S )Nzavg_pool2d: ú4 must either be a single int, or a tuple of two intsr)   r)   ©r2  r)   r*   rH   w	  rI   z1meta_avg_pool2d.<locals>.unpack.<locals>.<lambda>r   r’   ©r=   rK   r”   ©r2  r  ÚHÚWr)   rk  r*   Úunpackt	  s    

þzmeta_avg_pool2d.<locals>.unpackr6  ©r   r’   r­   c                   S   s   dS ©NzOavg_pool2d: stride must either be omitted, a single int, or a tuple of two intsr)   r)   r)   r)   r*   rH   €	  rI   z!meta_avg_pool2d.<locals>.<lambda>r   r’   r˜   rÇ  c                   S   s   dS ©Nzdivisor must be not zeror)   r)   r)   r)   r*   rH   �	  rI   rè  r]  éýÿÿÿr-  rq   rs   rú   )r=   rK   r”   r_   rv   Úpooling_output_shaper8   r:  Úpool2d_shape_checkrd   rD   r\   )r¥   r6  r˜   rÇ  r`  Úcount_include_padÚdivisor_overriderp  ÚkHÚkWÚdHÚdWÚpadHÚpadWrÔ  ra  ÚinputHeightÚ
inputWidthrb  rc  r}   rv   r)   r)   r*   Úmeta_avg_pool2dj	  sb    
	
þþ



ñür�  c                 C   sj   t | ||||||dd|	|
||||ƒ |  ¡ }|	}t|||d |ƒ t|||d |ƒ t|||d |ƒ d S )Nr’   rs   r­   )rv  r_   rh  )r¥   Z
gradOutputrÔ  ry  rz  r{  r|  r}  r~  ra  r  r€  rb  rc  Ú
mem_formatry   ÚnOutputPlaner)   r)   r*   Úavg_pool2d_backward_shape_check¸	  s,    ñr„  c                 C   s¶  t  t|ƒdkpt|ƒdkdd„ ¡ |d }t|ƒdkr<|n|d }	t  t|ƒdkpjt|ƒdkpjt|ƒdkdd„ ¡ t|ƒdkr„|n|d }
t|ƒdkrœ|	nt|ƒdkr¬|
n|d }t  t|ƒdkpÎt|ƒdkdd„ ¡ |d }t|ƒdkrð|n|d }t  |d k�p|dkdd„ ¡ |j}| ¡ d	k�r2|d
 nd}|d }|d }|d }t||||
d|ƒ}t||	||d|ƒ}t |¡}t|| |||	|
|||||||||ƒ t j	||j
|j|d�S )Nr’   r­   c                   S   s   dS )NzKavg_pool2d: kernel_size must either be a single int, or a tuple of two intsr)   r)   r)   r)   r*   rH   ò	  rI   z*meta_avg_pool2d_backward.<locals>.<lambda>r   c                   S   s   dS rr  r)   r)   r)   r)   r*   rH   ø	  rI   c                   S   s   dS )NzGavg_pool2d: padding must either be a single int, or a tuple of two intsr)   r)   r)   r)   r*   rH   þ	  rI   c                   S   s   dS rs  r)   r)   r)   r)   r*   rH   
  rI   rè  r]  rt  r-  rq   rú   )r=   rK   r”   rn   r_   ru  r8   r:  r„  rd   rD   r\   )ZgradOutput_r¥   r6  r˜   rÇ  r`  rw  rx  ry  rz  r{  r|  r}  r~  Ú
input_sizerÔ  ra  r  r€  rb  rc  r‚  r)   r)   r*   Úmeta_avg_pool2d_backwardä	  sj    þ"þ(þþ
ñür†  c                 C   s  t  t|ƒdkdd„ ¡ |d }t|ƒdkr0|n|d }t|ƒdkrH|n|d }	t  | pdt|ƒdkdd„ ¡ |sv|n|d }
|s†|nt|ƒdkr–|
n|d }|s¦|	nt|ƒdkr¶|
n|d }t  t|ƒdkdd„ ¡ |d }t|ƒdkrî|n|d }t|ƒdk�r|n|d }t  | jd	kd
d„ ¡ t  | �p8|dkdd„ ¡ |  d¡}|  d¡}|  d¡}|  d¡}|  d¡}t||||
d|ƒ}t||||d|ƒ}t||	||d|ƒ}t| ||||	|
|||||ddd||||||ddd� | jdk�rü|  ||||f¡S |  |||||f¡S d S )N©r’   rs   c                   S   s   dS ©NzFavg_pool3d: kernel_size must be a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   :
  rI   z!meta_avg_pool3d.<locals>.<lambda>r   r’   r­   c                   S   s   dS ©NzJavg_pool3d: stride must be omitted, a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   B
  rI   c                   S   s   dS ©NzBavg_pool3d: padding must be a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   J
  rI   ©rè  rù  c                   S   s   dS ©Nz9non-empty 4D or 5D (batch mode) tensor expected for inputr)   r)   r)   r)   r*   rH   R
  rI   c                   S   s   dS rs  r)   r)   r)   r)   r*   rH   W
  rI   r]  rt  r-  rq   zavg_pool3d()T)Úcheck_input_sizerè  )r=   rK   r”   ry   rv   ru  Úpool3d_shape_checkrm   )r¥   r6  r˜   rÇ  r`  rw  rx  ÚkTry  rz  ÚdTr{  r|  ÚpadTr}  r~  rÔ  ÚnslicesÚitimeÚiheightÚiwidthÚotimeÚoheightÚowidthr)   r)   r*   Úmeta_avg_pool3d-
  s€    
þþ  
þþþ




êr™  c                 C   sØ  t  t|ƒdkdd„ ¡ |d }t|ƒdkr0|n|d }	t|ƒdkrH|n|d }
t  | pdt|ƒdkdd„ ¡ |sv|n|d }|s†|	nt|ƒdkr–|n|d }|s¦|
nt|ƒdkr¶|n|d }t  t|ƒdkdd„ ¡ |d }t|ƒdkrî|n|d }t|ƒdk�r|n|d }t  |jd	kd
d„ ¡ t  | �p8|dkdd„ ¡ | d¡}| d¡}| d¡}| d¡}t||||d|ƒ}t||	||d|ƒ}t||
||d|ƒ}t|| |||	|
||||||||||||dƒ | |j¡S )Nr‡  c                   S   s   dS rˆ  r)   r)   r)   r)   r*   rH   ‘
  rI   z*meta_avg_pool3d_backward.<locals>.<lambda>r   r’   r­   c                   S   s   dS r‰  r)   r)   r)   r)   r*   rH   ™
  rI   c                   S   s   dS rŠ  r)   r)   r)   r)   r*   rH   ¡
  rI   r‹  c                   S   s   dS rŒ  r)   r)   r)   r)   r*   rH   ©
  rI   c                   S   s   dS rs  r)   r)   r)   r)   r*   rH   ®
  rI   r]  rt  r-  rq   zavg_pool3d_backward())	r=   rK   r”   ry   rv   ru  Úavg_pool3d_backward_shape_checkrm   rn   )rá  r¥   r6  r˜   rÇ  r`  rw  rx  r�  ry  rz  r�  r{  r|  r‘  r}  r~  r’  r“  r”  r•  Zotime_for_shape_checkZoheight_for_shape_checkZowidth_for_shape_checkr)   r)   r*   Úmeta_avg_pool3d_backwardƒ
  st    
þþ  
þþþ



ír›  c                    sZ   t  ˆ jdkpˆ jdk‡ fdd„¡ ˆ jd d… t|ƒ }t ˆ ¡}t j|ˆ jˆ j	|d�S )Nrs   rè  c                      s   dˆ j › �S )Nz"Expected 3D or 4D tensor, but got rz  r)   r‚   r)   r*   rH   ×
  rI   z*meta_adaptive_avg_pool2d.<locals>.<lambda>r-  rú   )
r=   rK   ry   rn   rJ   r8   r:  rd   rD   r\   )ro   Úoutput_sizerç   r}   r)   r‚   r*   Úmeta_adaptive_avg_pool2dÓ
  s    
þ
ür�  c                    s@   t  ˆ jdkpˆ jdk‡ fdd„¡ ˆ  ˆ jd d… t|ƒ ¡S )Nrè  rù  c                      s   dˆ j › �S )Nz"Expected 4D or 5D tensor, but got rz  r)   r‚   r)   r*   rH   é
  rI   z*meta_adaptive_avg_pool3d.<locals>.<lambda>rt  )r=   rK   ry   rm   rn   rJ   )ro   rœ  r)   r‚   r*   Úmeta_adaptive_avg_pool3då
  s
    
þrž  c                    sœ   ˆ j }td|ƒD ]$‰t ˆ  ˆ¡dk‡ ‡fdd„¡ qt |dkpH|dk‡fdd„¡ t ˆjˆ jk‡ ‡fdd„¡ tj}tˆƒrˆtj}ˆ 	ˆj
¡j|d	�S )
Nr’   r   c                      s   dˆ j › dˆ› d�S )Nz{adaptive_avg_pool2d_backward(): Expected grad_output to have non-zero                       size for non-batch dimensions, ú with dimension ú being emptyrz  r)   )Úgrad_outr§   r)   r*   rH   ô
  s   ÿÿz4meta__adaptive_avg_pool2d_backward.<locals>.<lambda>rs   rè  c                      s   dˆ j › �S )NzBadaptive_avg_pool2d_backward(): Expected 3D or 4D tensor, but got rz  r)   r‚   r)   r*   rH   ù
  rI   c                      s   dˆj › dˆ j › �S ©Nzexpected dtype z! for `grad_output` but got dtype rg   r)   )r¡  ro   r)   r*   rH   ý
  rI   r|   )ry   r–   r=   rK   rv   rD   r�   r=  r;  rm   rn   rÀ   )r¡  ro   ry   r}   r)   )r¡  r§   ro   r*   Ú"meta__adaptive_avg_pool2d_backwardî
  s$    þ
þ
þr£  c                 C   s   t | dƒ tj|tjd�S )NZadaptive_avg_pool3d_backwardr|   )Ú!_adaptive_pool_empty_output_checkr=   r€   rc  ©rá  ro   r)   r)   r*   Ú"meta__adaptive_avg_pool3d_backward  s    
r¦  )rá  r=  c                    s<   ˆj }td|ƒD ]&‰t ˆ ˆ¡dk‡ ‡‡fdd„¡ qd S )Nr’   r   c                      s   ˆ › dˆj › dˆ› d�S )Nzc(): Expected grad_output to have non-zero size for non-batch dimensions, but grad_output has sizes rŸ  r   rz  r)   ©r=  rá  r§   r)   r*   rH     s    z3_adaptive_pool_empty_output_check.<locals>.<lambda>)ry   r–   r=   rK   rv   )rá  r=  ry   r)   r§  r*   r¤    s    þr¤  r�   c                    s&  ˆj }t |dk‡fdd„¡ td|ƒD ]$‰ t ˆ ˆ ¡dk‡ ‡fdd„¡ q(t t|ƒdkdd„ ¡ d}d}d}ˆj d	krŽˆ d¡}|d7 }ˆ |d ¡}|\}}ˆj d
krÚ|||f}ˆ |¡}	ˆj|tjd�}
|	|
fS ||||f}t 	ˆ¡}ˆ |¡j
|d�}	ˆj|tjd�j
|d�}
|	|
fS d S )N©rs   rè  c                      s   dˆ j › �S )Nz:adaptive_max_pool2d(): Expected 3D or 4D tensor, but got: rz  r)   rX  r)   r*   rH     rI   z*meta_adaptive_max_pool2d.<locals>.<lambda>r’   r   c                      s   dˆj › dˆ › d�S )Nzjadaptive_max_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes rŸ  r   rz  r)   ©r§   r¥   r)   r*   rH   #  s    r­   c                   S   s   dS )NzCadaptive_max_pool2d(): internal error: output_size.size() must be 2r)   r)   r)   r)   r*   rH   +  rI   rè  rs   rg   r|   )ry   r=   rK   r–   rv   r”   rm   r‡   r8   r:  rÀ   )r¥   rœ  ry   ZdimHÚsizeBÚsizeDÚosizeHÚosizeWrz   r�   r…   r}   r)   r©  r*   Úmeta_adaptive_max_pool2d  sD    
þþ
þ





ÿr®  c                    sd   ˆ j }t |dk‡ fdd„¡ tˆ dƒ t ˆjˆ jk‡ ‡fdd„¡ t ˆ¡}ˆ ˆj¡j	|d�S )Nr¨  c                      s   dˆ j › �S )NzKadaptive_max_pooling2d_backward(): Expected 3D or 4D grad_output, but got: rz  r)   ©rá  r)   r*   rH   N  rI   z3meta_adaptive_max_pool2d_backward.<locals>.<lambda>Úadaptive_max_pool2d_backwardc                      s   dˆj › dˆ j › �S r¢  rg   r)   ©rá  r¥   r)   r*   rH   U  rI   r|   )
ry   r=   rK   r¤  rD   r8   r:  rm   rn   rÀ   )rá  r¥   r…   ry   r}   r)   r±  r*   Ú!meta_adaptive_max_pool2d_backwardH  s    
þ

þ
r²  c                    sæ   ˆj }t |dk‡fdd„¡ td|ƒD ]$‰ t ˆ ˆ ¡dk‡ ‡fdd„¡ q(t t|ƒdkdd„ ¡ d}d}d}|d	krŒˆ d¡}|d7 }ˆ |¡}|\}}}|d
kr¶||||f}	n|||||f}	ˆ |	¡}
ˆj|	tjd�}|
|fS )Nr‹  c                      s   dˆ j › �S )Nz:adaptive_max_pool3d(): Expected 4D or 5D tensor, but got: rz  r)   rX  r)   r*   rH   b  rI   z*meta_adaptive_max_pool3d.<locals>.<lambda>r’   r   c                      s   dˆj › dˆ › d�S )Nzjadaptive_max_pool3d(): Expected input to have non-zero size for non-batch dimensions, but input has sizes rŸ  r   rz  r)   r©  r)   r*   rH   g  s    rs   c                   S   s   dS )NzCadaptive_max_pool3d(): internal error: output_size.size() must be 3r)   r)   r)   r)   r*   rH   o  rI   rù  rè  rg   )ry   r=   rK   r–   rv   r”   rm   r‡   )r¥   rœ  ry   ZdimDrª  r«  ZosizeTr¬  r­  rz   r�   r…   r)   r©  r*   Úmeta_adaptive_max_pool3d\  s8    
þþ
þ



r³  c                 C   s   t | dƒ | |j¡S )NÚadaptive_max_pool3d_backward)r¤  rm   rn   )rá  r¥   r…   r)   r)   r*   Ú!meta_adaptive_max_pool3d_backwardˆ  s    
rµ  c                 C   s   |d krt dƒ‚|  |¡S )Nz:cannot repeat_interleave a meta tensor without output_size)r¿   rm   )Úrepeatsrœ  r)   r)   r*   Úmeta_repeat_interleave_Tensor�  s    r·  c                 C   s:   | j jst‚|j jst‚t| j|jƒ}| j|t| j ƒd�S r  )rD   r®   rb   r   rn   rm   r   )ÚrealÚimagrz   r)   r)   r*   Úmeta_complex–  s    rº  )Ú
fill_value)rv   r»  c                C   s   | j ||  ¡ ftjd�S r  )rm   r_   r=   rj   )ro   rv   r»  r)   r)   r*   Únonzero_staticŸ  s    r¼  c              
      s¬  t  tˆƒdd„ ¡ g }tˆƒD ]â\‰ ‰ˆd k	røt  ˆjt jt jt jt jfkdd„ ¡ ˆjt jt jfkrìˆ ¡ }t	|ƒ‰t  
ˆˆj ˆjk‡fdd„¡ tˆjƒD ]F‰t  
ˆjˆ ˆjˆˆ  k‡ ‡‡‡‡fdd„¡ | | dˆ¡¡ q¢n
| ˆ¡ q | ˆ¡ q |‰t  t	ˆƒˆjk‡‡fdd„¡ dd lm} t|jˆŽ ƒ‰t	ˆƒˆjk �r`ˆ d ¡ �qBd}d	}ˆD ]J‰|dk�rŠˆd k	�r´d}n*|dk�r¤ˆd k�r´d
}nˆd k	�rl �q¼�qld}|�s<g }g }tˆƒD ](\‰ ‰ˆd k	�rÒ| ˆ ¡ | ˆ¡ �qÒtˆƒD ](\‰ ‰ˆd k�r| ˆ ¡ | ˆ¡ �qˆ |¡‰|‰g }	g }
g }tˆƒD ]H\}‰ˆd k�rŒ|�rz|
 ˆj| ¡ n|	 ˆj| ¡ n
tˆjƒ}�qPˆ |	| |
 ¡S )Nc                   S   s   dS )Nz#at least one index must be providedr)   r)   r)   r)   r*   rH   §  rI   z#meta_index_Tensor.<locals>.<lambda>c                   S   s   dS )Nz?tensors used as indices must be long, int, byte or bool tensorsr)   r)   r)   r)   r*   rH   ¯  rI   c                      s   dˆ j › �S )Nú)too many indices for tensor of dimension r_  r)   r‚   r)   r*   rH   ¶  rI   c                	      s$   dˆj › dˆ › dˆj › dˆˆ › �S )NzThe shape of the mask z
 at index z0 does not match the shape of the indexed tensor rz  r)   )r§   ri   Újrå   ro   r)   r*   rH   »  rI   r’   c                      s   dˆj › dtˆ ƒ› d�S )Nr½  z (got rT   )ry   r”   r)   )r…   ro   r)   r*   rH   Æ  rI   r   Fr­   T)r=   rK   r  Ú	enumeraterD   rj   rw  rÑ   Únonzeror”   rk   ry   r–   rn   r—   ÚselectÚtorch._refsZ_refsr•   r    rš   rm   )ro   r…   rè   rÀ  ÚrefsÚstateZhas_contiguous_subspacer+  Ztransposed_indicesZbefore_shapeZafter_shapeZreplacement_shaper_   r)   )r§   ri   r…   r¾  rå   ro   r*   Úmeta_index_Tensor¥  s†    þ
þþþ











rÅ  c                 C   sT   d }d }d }|
d r"|   | ¡ ¡}|
d r8|   | ¡ ¡}|
d rJ|   |¡}|||fS )Nr   r’   r­   ©rm   rv   )Úgrad_output_Úinput_Úweight_Zbias_sizes_optr˜   rÇ  r/  Z
transposedr2  r1  Úoutput_maskZbackend_grad_inputZbackend_grad_weightZbackend_grad_biasr)   r)   r*   Úmeta_convolution_backward	  s    
rË  c                   sÔ   ˆ   d¡}ˆ  d¡}|  ||f¡} t ˆ  ¡ dkdd„ ¡ t ˆ ¡ dkdd„ ¡ t ˆ   d¡ˆ  d¡k‡ ‡fdd„¡ t ˆ   d¡ˆ  d¡k‡ ‡fd	d„¡ t |   d¡|ko¼|   d¡|kd
d„ ¡ |  |   ¡ ¡S )Nr’   r­   rs   c                   S   s   dS r  r)   r)   r)   r)   r*   rH   -  rI   zmeta_addbmm.<locals>.<lambda>c                   S   s   dS r  r)   r)   r)   r)   r*   rH   .  rI   r   c                      s   dˆ   d¡› dˆ  d¡› �S )Nz8batch1 and batch2 must have same number of batches, got r   rt   ru   r)   ©r  r  r)   r*   rH   1  rI   c                
      s6   dˆ   d¡› dˆ   d¡› dˆ  d¡› dˆ  d¡› d�	S )Nz#Incompatible matrix sizes for bmm (r’   r4   r­   rt   rT   ru   r)   rÌ  r)   r*   rH   5  s    c                   S   s   dS )Nz.self tensor does not match matmul output shaper)   r)   r)   r)   r*   rH   <  rI   )rv   r§  r=   rK   r_   rm   )ro   r  r  rÚ   rÙ   r  r  r)   rÌ  r*   Úmeta_addbmm'  s$    

þþþrÍ  )Ú
grad_scaleÚ	found_infc       	            s4   | |||||fD ]‰ t  tˆ tƒ‡ fdd„¡ qd S )Nc                      s   dt ˆ ƒ› �S ©Nz'exponent must be a tensor list but got ©rU   r)   ©Úlr)   r*   rH   W  rI   z#meta__fused_adam_.<locals>.<lambda>©r=   rK   rN   r   )ro   ÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚlrÚbeta1Úbeta2Úweight_decayÚepsÚamsgradÚmaximizerÎ  rÏ  r)   rÒ  r*   Úmeta__fused_adam_A  s
    
þrá  c       	            sZ   | |||||fD ]‰ t  tˆ tƒ‡ fdd„¡ qdd„ }|| ƒ||ƒ||ƒ||ƒ||ƒfS )Nc                      s   dt ˆ ƒ› �S rÐ  rÑ  r)   rÒ  r)   r*   rH   q  rI   z"meta__fused_adam.<locals>.<lambda>c                 S   s   dd„ | D ƒS )Nc                 S   s   g | ]}t  |¡‘qS r)   ©r=   r€   )r3   r9  r)   r)   r*   r7   u  s     z=meta__fused_adam.<locals>.empty_like_list.<locals>.<listcomp>r)   )Ztensor_listr)   r)   r*   Úempty_like_listt  s    z)meta__fused_adam.<locals>.empty_like_listrÔ  )ro   rÕ  rÖ  r×  rØ  rÙ  rÚ  rÛ  rÜ  rÝ  rÞ  rß  rà  rÎ  rÏ  rã  r)   rÒ  r*   Úmeta__fused_adam[  s    
þûrä  c                    s®   t  ˆ  ¡ dkdd„ ¡ t  ˆ ¡ dkdd„ ¡ t  ˆ jt jk‡ fdd„¡ t  ˆjt jk‡fdd„¡ t  ˆ  d¡ˆ d¡k‡ ‡fd	d„¡ ˆ jˆ  d¡ˆ d¡ft jd
�S )Nr­   c                   S   s   dS )Nza must be a 2D tensorr)   r)   r)   r)   r*   rH   ƒ  rI   zmeta__int_mm.<locals>.<lambda>c                   S   s   dS )Nzb must be a 2D tensorr)   r)   r)   r)   r*   rH   „  rI   c                      s   dˆ j › �S )Nzexpected self to be int8, got rg   r)   )r&  r)   r*   rH   ‡  rI   c                      s   dˆ j › �S )Nzexpected mat2 to be int8, got rg   r)   )r'  r)   r*   rH   ‹  rI   r’   r   c                
      s6   dˆ   d¡› dˆ   d¡› dˆ  d¡› dˆ  d¡› d�	S )Nz'Incompatible matrix sizes for _int_mm (r   r4   r’   rt   rT   ru   r)   r%  r)   r*   rH   �  s    rg   )r=   rK   r_   rD   rÑ   rv   rm   rÒ   r%  r)   r%  r*   Úmeta__int_mm€  s    

þ

þþrå  c                    sp   t  ˆ  ¡ dkdd„ ¡ t  ˆ jt jk‡ fdd„¡ ˆ  d¡}ˆ  d¡}ˆ j|d ||d  d	|d ft jd
�S )Nr­   c                   S   s   dS ©Nzw must be a 2D tensorr)   r)   r)   r)   r*   rH   ™  rI   z2meta__convert_weight_to_int4pack.<locals>.<lambda>c                      s   dˆ j › �S ©Nzexpected w to be int32, got rg   r)   ©rR  r)   r*   rH   œ  rI   r   r’   é   rá   é    rg   )r=   rK   r_   rD   rÒ   rv   rm   )rR  Zinner_k_tilesr³   rå   r)   rè  r*   Ú meta__convert_weight_to_int4pack—  s    

þ


üùrë  c                    s”   t  ˆ ¡ dkdd„ ¡ t  ˆ  ¡ dkdd„ ¡ t  ˆjt jt jt jfk‡fdd„¡ t  ˆ jt jk‡ fdd„¡ ˆjˆ 	d¡ˆ  	d¡d	 ˆjd
�S )Nr­   c                   S   s   dS ©Nzx must be a 2D tensorr)   r)   r)   r)   r*   rH   ­  rI   z*meta__weight_int4pack_mm.<locals>.<lambda>rè  c                   S   s   dS )Nzw must be a 4D tensorr)   r)   r)   r)   r*   rH   ®  rI   c                      s   dˆ j › �S ©Nz#expected x to be f32/f16/bf16, got rg   r)   r�   r)   r*   rH   ±  rI   c                      s   dˆ j › �S rç  rg   r)   rè  r)   r*   rH   µ  rI   r   ré  rg   )
r=   rK   r_   rD   râ   rã   rä   rÒ   rm   rv   )r4   rR  Zq_group_sizeZq_scale_and_zerosr)   ©rR  r4   r*   Úmeta__weight_int4pack_mm«  s    
þ

þrï  c                    s�   t  ˆ ¡ dkdd„ ¡ t  ˆjt jt jt jfk‡fdd„¡ t  ˆ  ¡ dkdd„ ¡ t  ˆ jt jk‡ fdd„¡ ˆjˆ 	d¡ˆ  	d¡ˆjd�S )	Nr­   c                   S   s   dS rì  r)   r)   r)   r)   r*   rH   ¼  rI   z*meta__weight_int8pack_mm.<locals>.<lambda>c                      s   dˆ j › �S rí  rg   r)   r�   r)   r*   rH   ¿  rI   c                   S   s   dS ræ  r)   r)   r)   r)   r*   rH   Á  rI   c                      s   dˆ j › �S )Nzexpected w to be int8, got rg   r)   rè  r)   r*   rH   Ä  rI   r   rg   )
r=   rK   r_   rD   râ   rã   rä   rÑ   rm   rv   )r4   rR  Zq_scalesr)   rî  r*   Úmeta__weight_int8pack_mmº  s    
þ

þrð  c           	         s  t  ˆ ¡ dk‡fdd„¡ t  ˆ ¡ dk‡fdd„¡ t  ˆ d¡ˆ d¡k‡‡fdd„¡ t  t ˆj¡dd„ ¡ t  t ˆj¡dd„ ¡ t  |d	kd
d„ ¡ t  ˆ dk‡ fdd„¡ ˆ d¡}ˆ d¡}ˆjd d… }ˆjd d… }tt  	||¡ƒ}| 
||g¡ ˆ |¡S )Nr­   c                      s   dˆ   ¡ › d�S )Nz1cdist only supports at least 2D tensors, X1 got: ÚDrr   r)   )Úx1r)   r*   rH   Í  rI   z$meta_cdist_forward.<locals>.<lambda>c                      s   dˆ   ¡ › d�S )Nz1cdist only supports at least 2D tensors, X2 got: rñ  rr   r)   )Úx2r)   r*   rH   Ñ  rI   rq   c                      s   dˆ   d¡› dˆ  d¡› �S )Nz4X1 and X2 must have the same number of columns. X1: rq   z X2: ru   r)   )rò  ró  r)   r*   rH   Õ  rI   c                   S   s   dS )Nz=cdist only supports floating-point dtypes, X1 got: {x1.dtype}r)   r)   r)   r)   r*   rH   Ù  rI   c                   S   s   dS )Nz=cdist only supports floating-point dtypes, X2 got: {x2.dtype}r)   r)   r)   r)   r*   rH   Ý  rI   r   c                   S   s   dS )Nz)cdist only supports non-negative p valuesr)   r)   r)   r)   r*   rH   ß  rI   ©Nr’   r­   c                      s
   dˆ › �S )Nz%possible modes: None, 1, 2, but was: r)   r)   )Úcompute_moder)   r*   rH   â  rI   r-  )r=   rK   r_   rv   r8   Úis_float_dtyperD   rn   r•   Úbroadcast_shapesÚextendrm   )	rò  ró  r  rõ  Úr1Úr2Úbatch_tensor1Úbatch_tensor2rç   r)   )rõ  rò  ró  r*   Úmeta_cdist_forwardÉ  s@    

þ

þþ
þ
þ
þ

rý  c                 C   s¼   |j d }|j d }|j d }|j d d… }|j d d… }	tt ||	¡ƒ}
|
 ¡ }| ||g¡ t |
¡}|dksŠ|dksŠ|dksŠ|dkr”t |¡S |t|j ƒkr¬| 	|¡}tj
|tjd�S )Nrq   r-  r   r|   )rn   r•   r=   r÷  Úcopyrø  ÚmathÚprodÚ
zeros_liker§  r€   r�   )r  rò  ró  r  ZcdistZc1rù  rú  rû  rü  r¤  Ztensor1_expand_sizeZbatch_productr)   r)   r*   Úmeta_cdist_backwardí  s    



 

r  c	                    sF  t  ˆ jt jt jfk‡ fdd„¡ t  ˆjt jt jfk‡fdd„¡ t  t ˆj¡‡fdd„¡ ˆ d¡}	|rŠt  |	dkdd„ ¡ |	d8 }	ˆ |	ˆ d¡¡}
t	dƒ\}}}ˆd k	�r"t  ||kd	d„ ¡ t  ˆjˆjk‡‡fd
d„¡ t  ˆj
dk‡fdd„¡ t  ˆ ¡ ˆ  ¡ k‡ ‡fdd„¡ ‡fdd„‰dd„ ‰‡‡fdd„}tˆƒdk�ršˆ  ˆ  d¡¡}ˆ  ˆ ¡ ¡}||k�rŽˆ  |	ˆ d¡¡}n
ˆ  d¡}n |ˆˆ|
|ƒ}|||fk�s¼|�sÎˆ ˆ  d¡¡}n
ˆ d¡}ˆ |	¡}ˆjd }||k�r,|�rt  |dkdd„ ¡ |d8 }ˆ |ˆjd ¡}nˆ | ¡ ¡}|
|||fS )Nc                      s   dˆ j › �S )Nz(expected indices to be long or int, got rg   r)   ©r…   r)   r*   rH     rI   z$meta_embedding_bag.<locals>.<lambda>c                      s   dˆ j › �S )Nz(expected offsets to be long or int, got rg   r)   )rõ   r)   r*   rH     rI   c                      s   dˆ j › �S )Nz/expected weight to be floating point type, got rg   r)   )rË   r)   r*   rH     rI   r   r’   c                   S   s   dS ©Nz1include_last_offset: numBags should be at least 1r)   r)   r)   r)   r*   rH   !  rI   rs   c                   S   s   dS )Nz@embedding_bag: per_sample_weights only supported with mode='sum'r)   r)   r)   r)   r*   rH   +  rI   c                      s   dˆj › dˆ j › d�S )Nzexpected weight (z) and per_sample_weights (z) to have same dtyperg   r)   )Úper_sample_weightsrË   r)   r*   rH   /  rI   c                      s   dˆ j › d�S )Nz1expected per_sample_weights to be 1D tensor, got rñ  r_  r)   )r  r)   r*   rH   3  rI   c                      s   dˆ  ¡ › dˆ   ¡ › d�S )Nz%expected per_sample_weights.numel() (z$ to be the same as indices.numel() (rT   ©rl   r)   )r…   r  r)   r*   rH   7  s    c                    s   ˆ | ||ƒo|  d¡dkS ©Nr   r’   ©r˜   ©rÂ   r   r«   Úpadding_idx)Úis_fast_path_index_selectr)   r*   Úis_fast_path_index_select_scale=  s    ÿz;meta_embedding_bag.<locals>.is_fast_path_index_select_scalec                 S   s<   | j tjks| j tjko:|  d¡dko:| d¡dko:|dk S ©Nr’   r   )rD   r=   r@   r>   r˜   )rÂ   r«   r
  r)   r)   r*   r  B  s    ÿþüz5meta_embedding_bag.<locals>.is_fast_path_index_selectc                    s&   |d k	rˆ| |||ƒS ˆ | ||ƒS d S r$   r)   r	  )r  r  r)   r*   Úis_fast_pathJ  s    z(meta_embedding_bag.<locals>.is_fast_pathÚcpuc                   S   s   dS r  r)   r)   r)   r)   r*   rH   d  rI   )r=   rK   rD   rj   rw  r8   rö  rv   rm   r–   ry   rl   rž  rn   )rË   r…   rõ   Úscale_grad_by_freqr�  Úsparser  Zinclude_last_offsetr
  Znum_bagsr«   ÚMODE_SUMÚ	MODE_MEANÚMODE_MAXr  Ú
offset2bagÚbag_sizeÚmax_indicesZfast_path_sumZnumBagsr)   )r…   r  r  rõ   r  rË   r*   Úmeta_embedding_bag  s~    
þ
þ

þ
þ
þ
þ
þþ




þr  c                 G   s@   t | ||f|žŽ \}}}}t|ƒdkr4| | ¡ ¡}||||fS )Nr  )r  rž  rm   rv   )rË   r…   rõ   r:   r«   r  r  r  r)   r)   r*   Úmeta_embedding_bag_forward_onlym  s      ÿÿr  c                 C   s.   |r|S | j js| j jr| j S |r(tjS | j S r$   )rD   r®   r©   r=   rj   )r¥   rD   Úpromote_int_to_longr)   r)   r*   Ú_get_reduction_dtypew  s    r  rg   c                C   s6   t | |dd�}t | j|¡}t| ||ƒ}| j||d�S )NT)r  rg   )r  r8   r  rn   r  rm   )r¥   r+  r  rD   rW  rç   r)   r)   r*   Úmeta_nansum„  s    r  c                 C   s$   t  | jtt|  ¡ ƒƒ¡}|  |¡S r$   )r8   r,  rn   rJ   r–   r_   rm   )r¥   rç   r)   r)   r*   Úmeta_median�  s
     ÿr  c                 C   sL   t | ƒdkrt d¡ t | j|f¡}t| ||ƒ}|  |¡| j|tjd�fS )Nr�  zmedian CUDA with indices outputrg   )	rž  r8   Úalert_not_deterministicr  rn   r  rm   r=   rj   )r¥   r_   r  rç   r)   r)   r*   Úmeta_median_mode_dim•  s    
þr  c                 C   s   | S r$   r)   r‚   r)   r)   r*   Úmeta_logical_not_«  s    r   c                    sd   t  tˆƒ|  ¡ kdd„ ¡ tˆƒ|  ¡  }d| t| jƒ ‰ ‡ ‡fdd„ttˆƒƒD ƒ}|  |¡S )Nc                   S   s   dS )NzZNumber of dimensions of repeat dims can not be smaller than number of dimensions of tensorr)   r)   r)   r)   r*   rH   ´  rI   zmeta_repeat.<locals>.<lambda>r[  c                    s   g | ]}ˆ | ˆ|  ‘qS r)   r)   r)  ©Zpadded_sizer¶  r)   r*   r7   »  s     zmeta_repeat.<locals>.<listcomp>)r=   rK   r”   r_   rJ   rn   r–   rm   )ro   r¶  Znum_new_dimensionsZtarget_sizer)   r!  r*   Úmeta_repeat°  s    þr"  c                 C   s   | S r$   r)   r‚   r)   r)   r*   Ú
meta_zero_¿  s    r#  c                 C   s   t |tjƒrt| j|jƒ | S r$   ©rN   r=   r
   rL   rn   ©ro   rx   r)   r)   r*   Úmeta_binop_inplaceÄ  s    r&  c                 C   s   t |tjƒrt| j|jƒ | S r$   r$  )ro   rx   rÙ   r)   r)   r*   Úmeta_binop_inplace_alphaÕ  s    	r'  c                 K   s   t | tjd�S ©Nr/   )r<   r   r9   )ro   Úkwargsr)   r)   r*   Ú
meta_roundã  s     ÿr*  c                    sj   t  t ˆj¡‡ ‡fdd„¡ tˆt jƒrJt  t ˆj¡‡ ‡fdd„¡ nt  tˆtƒ‡ ‡fdd„¡ d S )Nc                      s   ˆ › dˆj › �S )Nz7: Expected input tensor to have an integral dtype. Got rg   r)   )rE  ro   r)   r*   rH   í  rI   z#shift_dtype_check.<locals>.<lambda>c                      s   ˆ › dˆj › �S )Nz6: Expected shift value to have an integral dtype. Got rg   r)   ©rE  r  r)   r*   rH   ò  rI   c                      s   ˆ › dˆ› �S )Nz): Expected shift value to be an int. Got r)   r)   r+  r)   r*   rH   ÷  rI   )r=   rK   r8   r|  rD   rN   r
   r   ©rE  ro   r  r)   r,  r*   Úshift_dtype_checkê  s    
þ
þþr-  c                 C   s   t d| |ƒ t| |tjd�S )NÚrshiftr/   ©r-  r<   r   r9   r%  r)   r)   r*   Úmeta_rshiftsû  s      ÿr0  c                 C   s   t d| |ƒ t| |tjd�S )NÚlshiftr/   r/  r%  r)   r)   r*   Úmeta_lshifts  s      ÿr2  c                 C   s   |   | j¡S r$   rK  r‚   r)   r)   r*   Ú	meta_zero  s    r3  c                 C   s   | S r$   r)   ©ro   r  r)   r)   r*   Ú
meta_fill_  s    r5  c                 C   s
   t  | ¡S r$   râ  r4  r)   r)   r*   Ú	meta_fill  s    r6  c                 C   s   | S r$   r)   r‚   r)   r)   r*   Ú
meta_relu_  s    r7  c                 C   s
   t  | ¡S r$   râ  ©ro   r…   r„   Ú
accumulater)   r)   r*   Úmeta_index_put  s    r:  c                 C   s   t | j|jƒ | S r$   )rL   rn   )ro   r!  Úvaluer)   r)   r*   Úmeta_masked_fill_$  s    r<  c                 C   s    |   |  ¡ ¡jt | ¡d�}|S rî   )rm   rv   rÀ   r8   r:  )ro   r!  r   Zmasked_scaler)   r)   r*   Úmeta__masked_scale*  s    ÿr=  c                 C   s:   t  |jt jt jfkdd„ ¡ t  | j|jkdd„ ¡ | S )Nc                   S   s   dS )NzMask must be bool or uint8r)   r)   r)   r)   r*   rH   5  rI   z&meta_masked_scatter_.<locals>.<lambda>c                   S   s   dS )Nzdmasked_scatter: expected self and source to have same dtypes but got {self.dtype} and {source.dtype}r)   r)   r)   r)   r*   rH   9  rI   )r=   rK   rD   r  Úuint8)ro   r!  rë   r)   r)   r*   Úmeta_masked_scatter_2  s     ÿ
þr?  c                 C   s*   t | |ƒ\} }tj| tjd�}t|||ƒS rî   )r    r=   r€   r�   r?  )ro   r!  rë   r«   r)   r)   r*   Úmeta_masked_scatter?  s    r@  c                 C   s
   |   |¡S r$   r  )ro   r!  r‚  r)   r)   r*   Úmeta_masked_scatter_backwardG  s    rA  c                 C   s   | S r$   r)   r8  r)   r)   r*   Úmeta_index_put_L  s    rB  c                 C   s   |   | j¡S r$   )Úviewrn   r‚   r)   r)   r*   Ú
meta_aliasQ  s    rD  c                    sæ   t  |  ¡ dkdd„ ¡ t  | ¡ dkdd„ ¡ |  ¡ }| ¡ ‰ |d ‰|d ‰|d }ˆ d }ˆ||f‰t  ˆ d ˆko„ˆ d ˆk‡ ‡‡fdd„¡ | ˆ¡}|sâˆd k	rât  ˆ ¡ dkd	d„ ¡ t  ˆ ¡ ˆk‡‡fd
d„¡ |S )Nrs   c                   S   s   dS r  r)   r)   r)   r)   r*   rH   W  rI   z)common_meta_baddbmm_bmm.<locals>.<lambda>c                   S   s   dS r  r)   r)   r)   r)   r*   rH   X  rI   r   r­   r’   c                	      s&   dˆ› dˆ› dˆ d › dˆ d › d�	S r  r)   r)   r  r)   r*   rH   e  rI   c                   S   s   dS )Nzself must be a 3D tensorr)   r)   r)   r)   r*   rH   n  rI   c                      s   dˆ › dˆ  ¡ › �S )Nz*Expected an input tensor shape with shape z but got shape: ru   r)   )rœ  Úself_baddbmmr)   r*   rH   q  rI   )r=   rK   r_   rv   rm   )r  r  Zis_bmmrE  r  Zres_rowsZres_colsr«   r)   )r  r  r  rœ  rE  r*   Úcommon_meta_baddbmm_bmmV  s*    
þ

þrF  c                 C   s   t | |dƒS )NT)rF  )ro   rÖ   r)   r)   r*   Úmeta_bmmw  s    rG  c                 C   s<   | | }| | }|dkr8t |dk ƒt |dk ƒkr8|d8 }|S r  )r  )r4   ÚyÚqr�  r)   r)   r*   Údiv_rtn|  s
     rJ  c                 C   sZ   t | | | ||d   d |r(|d nd |ƒd }|rV|d | | | krV|d8 }|S r  )rJ  )Ú	inputSizeÚ
kernelSizerÎ  rÏ  r˜   r/  r`  Z
outputSizer)   r)   r*   Úpooling_output_shape_pad_lr†  s*    ÿþ
ýüûù	÷ÿrM  c                    sl   t  |dkdd„ ¡ t  ˆdk‡fdd„¡ t  ˆˆd ˆ  d d k‡ ‡‡fdd„¡ t| ˆˆˆ|ˆ |ƒS )Nr   c                   S   s   dS )Nzstride should not be zeror)   r)   r)   r)   r*   rH   œ  rI   z&pooling_output_shape.<locals>.<lambda>c                      s
   dˆ › �S )Nz'pad must be non-negative, but got pad: r)   r)   )Úpadr)   r*   rH   �  rI   r’   r­   c                      s   dˆ› dˆ› dˆ › �S )NzApad should be at most half of effective kernel size, but got pad=z, kernel_size=z and dilation=r)   r)   ©r/  rL  rN  r)   r*   rH      s    )r=   rK   rM  )rK  rL  rN  r˜   r/  r`  r)   rO  r*   ru  ›  s    þ      ÿru  c              	      sR  ˆ   ¡ }ˆ‰t ˆdkoˆdkdd„ ¡ t |dko:|dkdd„ ¡ t |dkoV|dkdd„ ¡ ˆ  d¡dkozˆ  d¡dk}|tjkr®t |dko¢|o¢ˆ  d	¡dkd
d„ ¡ nDt |d	krÌˆ  d¡dkrÌ|pä|dkoä|oäˆ  d	¡dk‡ fdd„¡ t ˆd ˆ
k�oˆd ˆ	k‡‡‡	‡
fdd„¡ t ˆdk�o6ˆdk‡‡‡‡‡‡fdd„¡ d S )Nr   c                   S   s   dS )NzCkernel size should be greater than zero, but got kH: {kH}, kW: {kW}r)   r)   r)   r)   r*   rH   À  rI   z$pool2d_shape_check.<locals>.<lambda>c                   S   s   dS )Nz>stride should be greater than zero, but got dH: {dH}, dW: {dW}r)   r)   r)   r)   r*   rH   Ä  rI   c                   S   s   dS )Nz\dilation should be greater than zero, but got dilationH: {dilationH}, dilationW: {dilationW}r)   r)   r)   r)   r*   rH   È  rI   r’   r­   rè  rs   c                   S   s   dS )NzŽExpected 4D (batch mode) tensor expected for input with channels_last layout with optional 0 dim batch size for input, but got: {input.size()}r)   r)   r)   r)   r*   rH   Ð  rI   c                      s   dˆ   ¡ › �S )NzYExpected 3D or 4D (batch mode) tensor with optional 0 dim batch size for input, but got: ru   r)   rX  r)   r*   rH   ×  rI   c                      s   dˆ› dˆ› dˆ› dˆ › �S )NzKpad should be smaller than or equal to half of kernel size, but got padW = z	, padH = z, kW = z, kH = r)   r)   )ry  rz  r}  r~  r)   r*   rH   Ü  rI   c                      s*   dˆ› dˆ › dˆ› dˆ› dˆ› dˆ› d�S ©NzGiven input size: (r4   z). Calculated output size: (z). Output size is too smallr)   r)   )r  r€  ra  rƒ  rb  rc  r)   r*   rH   â  rI   )r_   r=   rK   rv   r;  )r¥   ry  rz  r{  r|  r}  r~  Ú	dilationHÚ	dilationWra  r  r€  rb  rc  r}   ry   Z
valid_dimsr)   )r¥   r  r€  ry  rz  ra  rƒ  rb  rc  r}  r~  r*   rv  ª  sB    þþþ
þ
ýþþrv  )r¥   r’  r�  ry  rz  r�  r{  r|  ÚpTÚpHÚpWÚ	dilationTrQ  rR  r“  r”  r•  r–  r—  r˜  rE  r�  c              
      s   ˆ	j }t ˆdko ˆdko ˆdk‡‡‡fdd„¡ t ˆdkoLˆdkoLˆ dk‡ ‡‡fdd„¡ t ˆdkoxˆdkoxˆdk‡‡‡fdd„¡ t |dk‡‡	fdd„¡ t|ƒD ]8‰|dkrÂˆdkrÂq¬t ˆ	 ˆ¡dk‡‡‡	fd	d„¡ q¬|�r"t ˆ
ˆk�o
ˆˆk�o
ˆˆk‡‡
‡‡‡‡fd
d„¡ t ˆd ˆk�oLˆd ˆk�oLˆd ˆk‡‡‡‡‡‡fdd„¡ t ˆdk�o‚ˆdk�o‚ˆdk‡‡
‡‡‡‡‡fdd„¡ d S )Nr   c                      s   dˆ› dˆ › dˆ› �S )Nz5kernel size should be greater than zero, but got kT: z, kH: z, kW: r)   r)   )ry  r�  rz  r)   r*   rH     s    z$pool3d_shape_check.<locals>.<lambda>c                      s   dˆ› dˆ › dˆ› �S )Nz0stride should be greater than zero, but got dT: z, dH: z, dW: r)   r)   )r{  r�  r|  r)   r*   rH     s    c                      s   dˆ› dˆ › dˆ› �S )Nz9dilation should be greater than zero, but got dilationT: z, dilationH: z, dilationW: r)   r)   )rQ  rV  rR  r)   r*   rH     s    r‹  c                      s   ˆ › dˆj › �S )Nz/: Expected 4D or 5D tensor for input, but got: rz  r)   )rE  r¥   r)   r*   rH     rI   rù  c                      s   ˆ › dˆj › dˆ ˆ¡› d�S )NzZ: Expected input's non-batch dimensions to have positive length, but input has a shape of z and non-batch dimension z has length zero!)rn   rv   r)   ©rE  r§   r¥   r)   r*   rH   #  s    c                      s*   dˆ› dˆ › dˆ› dˆ› dˆ› dˆ› d�S )Nzinput image (T: rý  rî  z ) smaller than kernel size (kT: ú kH: ú kW: rT   r)   r)   )r”  r“  r•  ry  r�  rz  r)   r*   rH   -  s    r­   c                      s(   dˆ› dˆ› dˆ › dˆ› dˆ› dˆ› �S )NzHpad should be smaller than or equal to half of kernel size, but got kT: rY  rX  z padT: z padW: z padH: r)   r)   )ry  r�  rz  rT  rS  rU  r)   r*   rH   5  s    r’   c                      s6   dˆ› dˆ› dˆ › dˆ› dˆ› dˆ› dˆ› dˆ› d�S rP  r)   r)   )r”  r“  r•  r’  r—  r–  r˜  r)   r*   rH   =  s    )ry   r=   rK   r–   rv   )r¥   r’  r�  ry  rz  r�  r{  r|  rS  rT  rU  rV  rQ  rR  r“  r”  r•  r–  r—  r˜  rE  r�  ry   r)   )r{  r�  r|  rQ  rV  rR  rE  r§   r”  r¥   r“  r•  ry  r�  rz  r’  r—  r–  r˜  rT  rS  rU  r*   rŽ  è  sJ    þþþþþ	þ&þþrŽ  c                 C   sÊ   | j }t| |||||||	|
||||||||||||ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ d S )Nrè  rs   r­   r’   ©ry   rŽ  rh  )r¥   rá  r…   r’  r�  ry  rz  r�  r{  r|  rS  rT  rU  rV  rQ  rR  r“  r”  r•  r–  r—  r˜  rE  ry   r)   r)   r*   Úmax_pool3d_backward_shape_checkE  s@    ër[  )r¥   rá  r’  r�  ry  rz  r�  r{  r|  rS  rT  rU  r“  r”  r•  r–  r—  r˜  rE  c                 C   s„   | j }t| ||||||||	|
|ddd|||||||dƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ t|||d |ƒ d S )Nr’   Trè  rs   r­   rZ  )r¥   rá  r’  r�  ry  rz  r�  r{  r|  rS  rT  rU  r“  r”  r•  r–  r—  r˜  rE  ry   r)   r)   r*   rš  ƒ  s:    êrš  c                 C   sB  dd„ }|d|ƒ\}}t  t|ƒdkdd„ ¡ t|ƒdkrF|| }	}
n|d|ƒ\}	}
|d	|ƒ\}}|d
|ƒ\}}|  d¡}|  d¡}|  d¡}t | ¡}|t jkr¼t  |  ¡ dkdd„ ¡ n4|t jkràt  |  ¡ dkdd„ ¡ nt  ddd„ ¡ t	||||	||ƒ}t	||||
||ƒ}t
| |||	|
||||||||||ƒ |||fS )Nc                    sD   t  t|ƒdk‡ fdd„¡ |d }t|ƒdkr4|n|d }||fS )Nri  c                      s   dˆ › d�S )Nzmax_pool2d: rj  r)   r)   rk  r)   r*   rH   À  rI   zEmax_pool2d_checks_and_compute_shape.<locals>.unpack.<locals>.<lambda>r   r’   rl  rm  r)   rk  r*   rp  ½  s    

þz3max_pool2d_checks_and_compute_shape.<locals>.unpackr6  rq  c                   S   s   dS )NzOmax_pool2d: stride must either be omitted, a single int, or a tuple of two intsr)   r)   r)   r)   r*   rH   Ê  rI   z5max_pool2d_checks_and_compute_shape.<locals>.<lambda>r   r˜   rÇ  r/  rt  r-  rq   rè  c                   S   s   dS )NzMnon-empty 4D (batch mode) tensor expected for input with channels_last layoutr)   r)   r)   r)   r*   rH   Û  rI   r¨  c                   S   s   dS )Nz9non-empty 3D or 4D (batch mode) tensor expected for inputr)   r)   r)   r)   r*   rH   à  rI   Fc                   S   s   dS )Nz?Unsupport memory format. Supports only ChannelsLast, Contiguousr)   r)   r)   r)   r*   rH   å  rI   )r=   rK   r”   rv   r8   r:  r;  r_   r�   ru  rv  )r¥   r6  r˜   rÇ  r/  r`  rp  ry  rz  r{  r|  r}  r~  rQ  rR  ra  r  r€  r}   rb  rc  r)   r)   r*   r^  ¹  sb    	
þ





þ

þþñr^  c                    s„   t ˆ|||||ƒ\}‰‰t ˆjˆ jk‡ ‡fdd„¡ |‰ˆj‰‡‡‡‡fdd„}	|	ˆ ƒ |	|ƒ t ˆ¡}
tjˆjˆjˆj	|
d�S )Nc                      s   dˆj › dˆ j › �S )NzExpected dtype z  for `gradOutput` but got dtype rg   r)   r¥  r)   r*   rH     rI   z7meta_max_pool2d_with_indices_backward.<locals>.<lambda>c                    s:   t | ˆˆd ˆ ƒ t | ˆˆd ˆƒ t | ˆˆd ˆƒ d S )Nrs   r­   r’   )rh  )r9  )rƒ  ry   rb  rc  r)   r*   Ú_check_dim_size  s    z>meta_max_pool2d_with_indices_backward.<locals>._check_dim_sizerú   )
r^  r=   rK   rD   ry   r8   r:  rd   rn   r\   )rá  ro   r6  r˜   rÇ  r/  r`  r…   ra  r\  r}   r)   )rá  rƒ  ry   rb  rc  ro   r*   Ú%meta_max_pool2d_with_indices_backward   s8         ÿü
þ
ür]  c                 C   sŒ   t | |||||ƒ\}}}|  ¡ dkr.|  d¡nd}	t | ¡}
|  ¡ dkrT|||g}n|	|||g}tj|| j| j|
d�tj|tj	| j|
d�fS r\  )
r^  r_   rv   r8   r:  r=   rd   rD   r\   r‡   r_  r)   r)   r*   Úmeta_max_pool2d_with_indices,  s<         ÿü
üüùr^  c           
   	      s   t  ˆjdk‡fdd„¡ ˆj}t|d |ƒD ],‰ t  ˆ ˆ ¡dkdˆ ¡ › dˆ › d�¡ q.t  tˆƒd	kd
d„ ¡ t  t|ƒd	kdd„ ¡ ˆ d¡}ˆ d¡‰ˆ d¡‰|dkr¾ˆ d¡}nd}t  ˆjˆjkdd„ ¡ t  ˆjdk‡fdd„¡ ˆ d¡}ˆ d¡}ˆ d	¡‰ t  ||kd¡ t  ||kdd„ ¡ t  ˆ d	k‡ fdd„¡ t  |d ˆd  d ˆk‡‡fdd„¡ t  |d ˆd  d ˆk‡‡fdd„¡ ˆ ¡ dk�rÆ|||d |d g}	n||d |d g}	t j|	ˆjˆj	d�t j|	t j
ˆj	d�fS )Nr¨  c                      s   dˆ j › �S )Nz:fractional_max_pool2d: Expected 3D or 4D tensor, but got: r_  r)   ©Úself_r)   r*   rH   R  rI   z,meta_fractional_max_pool2d.<locals>.<lambda>rs   r   z^fractional_max_pool2d: Expected input to have non-zero  size for non-batch dimenions, but got rŸ  z emptyr­   c                   S   s   dS )NzNfractional_max_pool2d: kernel_size musteither be a single int or tuple of Intsr)   r)   r)   r)   r*   rH   `  rI   c                   S   s   dS )NzOfractional_max_pool2d: output_size must either be a single int or tuple of Intsr)   r)   r)   r)   r*   rH   e  rI   rt  r-  rq   rè  r’   c                   S   s   dS )Nz6Expect _random_samples to have the same dtype as inputr)   r)   r)   r)   r*   rH   s  rI   c                      s   dˆ j › �S )Nz1Expect _random samples to have 3 dimensions got, r_  r)   )Úrandom_samplesr)   r*   rH   w  rI   z=Expect _random_samples.size(0) no less then input batch size.c                   S   s   dS )Nz<Expect _random_samples.size(1) equals to input channel size.r)   r)   r)   r)   r*   rH   ƒ  rI   c                      s   dˆ › d�S )Nz/Expect _random_samples.size(2) equals to 2 got Ú.r)   r)   )r¡   r)   r*   rH   …  rI   c                      s   dˆd › dˆ › �S )Nz%fractional_max_pool2d: kernel height r   z' is too large relative to input height r)   r)   )Úinput_heightr6  r)   r*   rH   ‰  rI   c                      s   dˆd › dˆ › �S )Nz$fractional_max_pool2d: kernel width r’   z& is too large relative to input width r)   r)   )Úinput_widthr6  r)   r*   rH   �  rI   ©rD   r\   )r=   rK   ry   r–   rv   r”   rD   r_   rd   r\   r‡   )
r`  r6  rœ  ra  ry   Zinput_channelsZinput_batchr³   Úcrv   r)   )r¡   rc  rd  r6  ra  r`  r*   Úmeta_fractional_max_pool2dN  s„    
þþ
þ
þ



þ
þ


þþþþýýúrg  c           	         s  t  d¡ t ˆjtjk‡fdd„¡ t tˆƒdk‡fdd„¡ ˆ\}}t ˆjdk‡fdd„¡ t ˆjˆjk‡‡fdd„¡ t	d	ˆjƒD ]$‰ t ˆ 
ˆ ¡d
k‡ ‡fdd„¡ qŽˆ ¡ }ˆjdkrâ| 
d
¡}| |||f¡}n&| 
d
¡}| 
d	¡}| ||||f¡}|S )NZmax_unpooling2d_forward_outc                      s   dˆ j › �S )Nz2elements in indices should be type int64 but got: rg   r)   r  r)   r*   rH   ª  rI   z#meta_max_unpool2d.<locals>.<lambda>r­   c                      s   dt ˆ ƒ› d�S )NzMThere should be exactly two elements (height, width) in output_size, but got ú
 elements.rÆ  r)   ©rœ  r)   r*   rH   ®  s    r¨  c                      s   dˆ j › d�S )NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with ú dimensions.r_  r)   r_  r)   r*   rH   ¸  s    c                      s   dˆj › dˆ j › �S ©NzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: rz  r)   )r…   r`  r)   r*   rH   ¿  s    r’   r   c                      s   dˆj › dˆ › d�S )NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got rŸ  ú being empty.rz  r)   )r§   r`  r)   r*   rH   È  s    rs   )r8   r  r=   rK   rD   r‡   r”   ry   rn   r–   rv   rŠ   rm   )	r`  r…   rœ  r—  r˜  ro   Ú	nchannelsrè   rÔ  r)   )r§   r…   rœ  r`  r*   Úmeta_max_unpool2d£  s>    


þ

þ
þ
þþ	



rn  c                    s  t  ˆjt jkdd„ ¡ t  ˆjdk‡fdd„¡ t  tˆƒdk‡fdd„¡ t  tˆƒdk‡fdd„¡ t  tˆƒdk‡fdd„¡ t  ˆjˆjk‡‡fd	d„¡ td
ˆjƒD ]&‰t  ˆ ˆ¡dk‡ ‡‡fdd„¡ q°t  ˆd dkoþˆd
 dkoþˆd dk‡fdd„¡ d S )Nc                   S   s   dS )Nz(elements in indices should be type int64r)   r)   r)   r)   r*   rH   Þ  rI   z._max_unpooling3d_shape_check.<locals>.<lambda>r‹  c                      s   dˆ j › d�S )NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with rj  r_  r)   rX  r)   r*   rH   â  rI   rs   c                      s   dt ˆ ƒ› d�S )NzVThere should be exactly three elements (depth, height, width) in output_size, but got rh  rÆ  r)   ri  r)   r*   rH   æ  s    c                      s   dt ˆ ƒ› d�S )NzRThere should be exactly three elements (depth, height, width) in stride, but got: rh  rÆ  r)   r  r)   r*   rH   í  rI   c                      s   dt ˆ ƒ› d�S )NzSThere should be exactly three elements (depth, height, width) in padding, but got: rh  rÆ  r)   )rÇ  r)   r*   rH   ñ  rI   c                      s   dˆj › dˆ j › �S rk  rz  r)   )r…   r¥   r)   r*   rH   õ  s    r’   r   c                      s   ˆ › dˆj › dˆ› d�S )NzI: Expected input to have non-zero size for non-batch dimensions, but got rŸ  rl  rz  r)   rW  r)   r*   rH   þ  s    r­   c                      s
   dˆ › �S )Nz5strides should be greater than zero, but got stride: r)   r)   r  r)   r*   rH     rI   )	r=   rK   rD   r‡   ry   r”   rn   r–   rv   )r¥   r…   rœ  r˜   rÇ  rE  r)   )rE  r§   r…   r¥   rœ  rÇ  r˜   r*   Ú_max_unpooling3d_shape_checkÜ  sB    
 ÿ
þ

þ

þ

þ
þþ	"
þro  c                 C   s‚   t  d¡ t| ||||dƒ |  ¡ }|\}}}| jdkrV| d¡}	| |	|||f¡}
n(| d¡}| d¡}	| ||	|||f¡}
|
S )NZmax_unpooling3d_forward_outzmax_unpooling3d()rè  r   r’   )r8   r  ro  rŠ   ry   rv   rm   )r`  r…   rœ  r˜   rÇ  ro   Zodepthr—  r˜  rm  rè   rÔ  r)   r)   r*   Úmeta_max_unpool3d  s$    
     ÿ




rp  c                 C   sÎ  t  t|ƒdkdd„ ¡ |d }t|ƒdkr0|n|d }t|ƒdkrH|n|d }t  | pdt|ƒdkdd„ ¡ |sv|n|d }	|s†|nt|ƒdkr–|	n|d }
|s¦|nt|ƒdkr¶|	n|d }t  t|ƒdkdd„ ¡ |d }t|ƒdkrî|n|d }t|ƒdk�r|n|d }t  t|ƒdkd	d„ ¡ |d }t|ƒdk�rB|n|d }t|ƒdk�r\|n|d }t  | jd
kdd„ ¡ | jdk�r�|  d¡nd}|  d¡}|  d¡}|  d¡}|  d¡}t||||	||ƒ}t||||
||ƒ}t||||||ƒ}t| |||||	|
|||||||||||||dƒ | jdk�o<t | ¡t j	k}| jdk�r||  
d¡}| ¡  �ol|jt j	d�}||||f}n|||||f}|  |¡}| j|t jd�}|�rÆ|jt j	d�}|jt j	d�}||fS )Nr‡  c                   S   s   dS ©NzMmax_pool3d: kernel_size must either be a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   /  rI   z.meta_max_pool3d_with_indices.<locals>.<lambda>r   r’   r­   c                   S   s   dS ©NzQmax_pool3d: stride must either be omitted, a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   7  rI   c                   S   s   dS ©NzImax_pool3d: padding must either be a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   ?  rI   c                   S   s   dS ©NzJmax_pool3d: dilation must be either a single int, or a tuple of three intsr)   r)   r)   r)   r*   rH   G  rI   r‹  c                   S   s   dS rŒ  r)   r)   r)   r)   r*   rH   O  rI   rù  éûÿÿÿr]  rt  r-  rq   zmax_pool3d_with_indices()rè  r|   rg   )r=   rK   r”   ry   rv   ru  rŽ  r8   r:  rE  r­  rµ  rm   r‡   rÀ   )r¥   r6  r˜   rÇ  r/  r`  r�  ry  rz  r�  r{  r|  rS  rT  rU  rV  rQ  rR  rÔ  r’  r“  r”  r•  r–  r—  r˜  r;  Úinput_channels_last_checkrz   r�   r…   r)   r)   r*   Úmeta_max_pool3d_with_indices#  s¢    

þþ  
þ
þþ



ëÿ
ÿÿþ
rw  c                 C   sd  t  t|ƒdkdd„ ¡ |d }t|ƒdkr0|n|d }	t|ƒdkrH|n|d }
t  | pdt|ƒdkdd„ ¡ |sv|n|d }|s†|	nt|ƒdkr–|n|d }|s¦|
nt|ƒdkr¶|n|d }t  t|ƒdkdd„ ¡ |d }t|ƒdkrî|n|d }t|ƒdk�r|n|d }t  t|ƒdkd	d„ ¡ |d }t|ƒdk�rB|n|d }t|ƒdk�r\|n|d }t  |jd
kdd„ ¡ | d¡}| d¡}| d¡}| d¡}|  d¡}|  d¡}|  d¡}t|| ||||	|
|||||||||||||||dƒ |jdk�ot |¡t jk}|jdk�r@| 	d¡}| 
¡  �o>|j
t jd�}| |j¡}|�r`|jt jd�}|S )Nr‡  c                   S   s   dS rq  r)   r)   r)   r)   r*   rH   š  rI   z7meta_max_pool3d_with_indices_backward.<locals>.<lambda>r   r’   r­   c                   S   s   dS rr  r)   r)   r)   r)   r*   rH   ¢  rI   c                   S   s   dS rs  r)   r)   r)   r)   r*   rH   ª  rI   c                   S   s   dS rt  r)   r)   r)   r)   r*   rH   ²  rI   r‹  c                   S   s   dS rŒ  r)   r)   r)   r)   r*   rH   º  rI   r]  rt  r-  rq   z"max_pool3d_with_indices_backward()rù  rè  r|   )r=   rK   r”   ry   rv   r[  r8   r:  rE  r­  rµ  rm   rn   rÀ   )rá  r¥   r6  r˜   rÇ  r/  r`  r…   r�  ry  rz  r�  r{  r|  rS  rT  rU  rV  rQ  rR  r’  r“  r”  r•  r–  r—  r˜  r;  rv  rã  r)   r)   r*   Ú%meta_max_pool3d_with_indices_backwardŒ  sœ    
þþ  
þ
þþ






éÿ
ÿÿþrx  ©r¥   Úgridc                    sÊ   t  ˆjˆ jk‡ ‡fdd„¡ t  ˆjt jko8ˆ jt jk‡ ‡fdd„¡ t  ˆjd ˆ jd k‡ ‡fdd„¡ t  ˆ jd ˆjd k‡ ‡fdd„¡ tdˆjƒD ]$‰t  ˆjˆ dk‡‡fd	d„¡ q d S )
Nc                      s   dˆj › dˆ j › �S )NzNgrid_sampler(): expected input and grid to be on same device, but input is on z and grid is on r  r)   ©rz  r¥   r)   r*   rH   ö  s    z+check_grid_sampler_common.<locals>.<lambda>c                      s   dˆj › dˆ j › �S )NzTgrid_sampler(): expected input and grid to have torch.strided layout, but input has z and grid has )r[   r)   r{  r)   r*   rH   ý  s    r   c                      s   dˆj › dˆ j › �S )NzZgrid_sampler(): expected grid and input to have same batch size, but got input with sizes ú and grid with sizes rz  r)   r{  r)   r*   rH     s    rq   r­   c                      s   dˆj d › dˆ j› �S )Nz+grid_sampler(): expected grid to have size r­   z, in last dimension, but got grid with sizes )ry   rn   r)   r{  r)   r*   rH     s    c                      s   dˆj › dˆ › d�S )NzYgrid_sampler(): expected input to have non-empty spatial dimensions, but input has sizes rŸ  r   rz  r)   r©  r)   r*   rH     s    )r=   rK   r\   r[   r»  rn   ry   r–   ry  r)   )rz  r§   r¥   r*   Úcheck_grid_sampler_commonó  s*    
þþþþþr}  c                   @   s   e Zd ZdZdZdZdS )ÚGridSamplerInterpolationr   r’   r­   N)rV   Ú
__module__Ú__qualname__ZBILINEARZNEARESTÚBICUBICr)   r)   r)   r*   r~    s   r~  ©r¥   rz  Úinterpolation_modec                    sP   t  ˆjdkoˆjˆ jk‡ ‡fdd„¡ t  ˆjdko@|tjjk dd„ ¡ d S )Nrù  c                      s   dˆj › dˆ j › �S )Nzdgrid_sampler(): expected 5D input and grid with same number of dimensions, but got input with sizes r|  rz  r)   r{  r)   r*   rH   $  s    z'check_grid_sampler_3d.<locals>.<lambda>c                   S   s   dS )Nz<grid_sampler(): bicubic interpolation only supports 4D inputr)   r)   r)   r)   r*   rH   /  rI   )r=   rK   ry   r~  r�  r;  r‚  r)   r{  r*   Úcheck_grid_sampler_3d!  s    þ

þûr„  c           
      C   s:   |d }|rt j|t jd�}nd }t j|t jd�}	||	fS ©Nr   r|   )r=   r  r�   r€   ©
rá  r¥   rz  rƒ  Úpadding_modeÚalign_cornersrÊ  Zinput_requires_gradrã  Ú	grad_gridr)   r)   r*   Úgrid_sampler_2d_backward_meta3  s    
rŠ  c           
      C   s\   t | |ƒ t| ||ƒ | jd }| jd }|jd }|jd }|jd }	|  |||||	f¡S )Nr   r’   r­   rs   )r}  r„  rn   rm   )
r¥   rz  rƒ  r‡  rˆ  r$  ÚCZout_DZout_HZout_Wr)   r)   r*   Úgrid_sampler_3dF  s    	





rŒ  r‰  c           
      C   sP   t ||ƒ t|||ƒ |d }|r4tj|tjd�}nd }tj|tjd�}	||	fS r…  )r}  r„  r=   r  rc  r€   r†  r)   r)   r*   Úgrid_sampler_3d_backwardY  s    
 ÿr�  c                 O   s4   |  dd ¡}|st |¡}||d< tj| f|ž|ŽS )NrD   )rC   r8   Z	get_dtyper=   rd   )rv   r»  r:   r)  rD   r)   r)   r*   Úfullq  s
    
rŽ  c                 C   sº   |t jkr”t  |d kdd„ ¡ t jd|d kr2| jn|||d krD| jn||d�}| jrp| |  ¡ |  	¡ |  
¡ ¡ n| |  ¡ |  ¡ d¡ | d¡ |S tjj| |||||d�}| d¡ |S )Nc                   S   s   dS )Nz9memory format option is only supported by strided tensorsr)   r)   r)   r)   r*   rH   ‡  rI   zzeros_like.<locals>.<lambda>r   rµ   Tr  )r=   Z
sparse_coorK   rd   rD   r\   Ú	is_sparseZsparse_resize_and_clear_rv   Ú
sparse_dimÚ	dense_dimr_   Z_coalesced_r"   r€   rÁ   Úfill_)ro   rD   r[   r\   r]   r}   r±  r)   r)   r*   r  {  s>    	
þû  ÿ
ú	
r  c                    s¼   ˆ  ¡ }t |dkdd„ ¡ ˆ dkr(ˆ nˆ | ‰ ˆ ˆ ¡}t ˆ |kpNˆ|k ‡ ‡‡fdd„¡ ˆdkrnˆnˆ| ‰tˆ ¡ ƒ}tˆ ¡ ƒ}ˆ ¡ ˆ|ˆ    }|ˆ = |ˆ = ˆ |||¡S )Nr   c                   S   s   dS )Nz-select() cannot be applied to a 0-dim tensor.r)   r)   r)   r)   r*   rH   ­  rI   zmeta_select.<locals>.<lambda>c                      s   dˆ› dˆ  ¡ › dˆ › �S )Nzselect(): index z! out of range for tensor of size z at dimension ru   r)   ©r_   ri   ro   r)   r*   rH   µ  rI   )r_   r=   rk   rv   r•   r˜   rœ   r›   )ro   r_   ri   ry   rv   Znew_sizerÇ   Znew_storage_offsetr)   r“  r*   Úmeta_select¨  s$    þ
þr”  c                 C   s
   t  | ¡S r$   ©r8   Zclone_preserve_strides)ro   rÂ   r_   ri   r)   r)   r*   Úmeta_select_scatterÅ  s    r–  c                 C   s
   t  | ¡S r$   r•  )ro   rÂ   r_   rX   rW   Ústepr)   r)   r*   Úmeta_slice_scatterÊ  s    r˜  )r_   Údim_post_exprÚwrap_scalarc                 C   sb   |dkr|st ‚d}| }|d }| |k s2| |krNt d| › d|› d|› d�ƒ‚| dk r^| |7 } | S )Nr   r’   zdim z out of bounds (rS   rT   )rb   )r_   r™  rš  r  r  r)   r)   r*   rˆ   Ð  s    ,rˆ   c                 C   s   |   ¡ dkrdS | j| S r  re  )r9  r_   r)   r)   r*   Úensure_nonempty_sizeÜ  s    r›  c                    st   t ˆ ¡ dƒ}t ˆ ¡ dƒ}t ||kdd„ ¡ t|ƒD ]6‰ˆˆ kr8t tˆˆƒtˆˆƒk‡ ‡‡‡fdd„¡ q8d S )Nr’   c                   S   s   dS )NzDIndex tensor must have the same number of dimensions as input tensorr)   r)   r)   r)   r*   rH   æ  rI   z$gather_shape_check.<locals>.<lambda>c                      s$   dˆ› dˆj › �dˆj › dˆ › � S )Nz!Size does not match at dimension z expected index ú to be smaller than self ú apart from dimension rz  r)   ©r_   r§   ri   ro   r)   r*   rH   ì  s   ÿ)r  r_   r=   rK   r–   r›  )ro   r_   ri   Ú	self_dimsZ
index_dimsr)   rž  r*   Úgather_shape_checká  s    þþr   c                    sb   ddl m} t||  ¡ ƒ}|ˆ  ¡ dkƒ}|sVt ˆ jtjk‡ fdd„¡ t	| |ˆ ƒ |  
ˆ j¡S )Nr   ©Úguard_size_obliviousc                      s   dˆ j › �S )Nz2gather(): Expected dtype int64 for index, but got rg   r)   rh   r)   r*   rH   ú  rI   zmeta_gather.<locals>.<lambda>)r   r¢  rˆ   r_   rl   r=   rK   rD   rj   r   rm   rn   )ro   r_   ri   Zsparse_gradr¢  Úwrapped_dimZis_index_emptyr)   rh   r*   Úmeta_gatherñ  s    

þr¤  c                 C   s„   |rT| dkrdS | dkrdS | dkr(dS | dkr4dS | d	kr@d
S t  ddd„ ¡ d S | dkr`dS | dkrldS t  ddd„ ¡ d S d S )NÚsumZ
REDUCE_ADDr   ZREDUCE_MULTIPLYZmeanZREDUCE_MEANZamaxZREDUCE_MAXIMUMZaminZREDUCE_MINIMUMFc                   S   s   dS )Nz=reduce argument must be either sum, prod, mean, amax or amin.r)   r)   r)   r)   r*   rH     rI   z#get_operator_enum.<locals>.<lambda>ÚaddÚmultiplyc                   S   s   dS )Nz/reduce argument must be either add or multiply.r)   r)   r)   r)   r*   rH     rI   r‘  )Úreduce_Úuse_new_optionsr)   r)   r*   Úget_operator_enum  s,    þrª  c                    s`   ddl m} || ¡ dkƒr8t |jtjk‡ fdd„¡ |d k	r\t |j|jk‡ fdd„¡ d S )Nr   r¡  c                      s
   ˆ › d�S )Nz"(): Expected dtype int64 for indexr)   r)   ©Úmethod_namer)   r*   rH   "  rI   z,scatter_gather_dtype_check.<locals>.<lambda>c                      s
   ˆ › d�S )Nz0(): Expected self.dtype to be equal to src.dtyper)   r)   r«  r)   r*   rH   (  rI   )r   r¢  rl   r=   rK   rD   rj   )r¬  ro   ri   Úsrc_optr¢  r)   r«  r*   Úscatter_gather_dtype_check  s    

þ

þr®  c                 C   s
   t | dƒS rÄ   )r  rr   r)   r)   r*   Úensure_nonempty_dim,  s    r¯  c           	         s0  ddl m} |ˆ ¡ dkƒr d S t tˆ ¡ ƒtˆ ¡ ƒkdd„ ¡ d}tˆ ¡ ƒ}t|ƒD ].}tˆ|ƒ}|ˆ krtq\|tˆ|ƒkr\d} qŒq\|sÆˆd k	rÆt|ƒD ]$}tˆ|ƒ}|tˆ|ƒkr d} qÆq ˆd k	�rt tˆ ¡ ƒtˆ ¡ ƒkdd„ ¡ t | ‡ ‡‡‡fdd„¡ nt | ‡ ‡‡fd	d„¡ d S )
Nr   r¡  c                   S   s   dS ©NzCIndex tensor must have the same number of dimensions as self tensorr)   r)   r)   r)   r*   rH   8  rI   z%scatter_shape_check.<locals>.<lambda>FTc                   S   s   dS r°  r)   r)   r)   r)   r*   rH   R  rI   c                      s&   dˆj › dˆj › �dˆ › dˆj › � S )NúExpected index rœ  r�  z and to be smaller than src rz  r)   ©r_   ri   ro   r­  r)   r*   rH   V  s   ÿc                      s   dˆj › dˆj › �dˆ › � S )Nr±  rœ  r�  rz  r)   r“  r)   r*   rH   \  s   ÿ)	r   r¢  rl   r=   rK   r¯  r_   r–   r›  )	ro   r_   ri   r­  r¢  Zis_wrong_shaperŸ  r¡   Zindex_d_sizer)   r²  r*   Úscatter_shape_check1  sF    þ


þþþr³  c                 C   s@   t ||  ¡ ƒ}td| ||ƒ t| |||ƒ |d k	r<t||ƒ d S )NÚscatter)rˆ   r_   r®  r³  rª  )ro   r_   ri   rÂ   r¨  r©  r£  r)   r)   r*   Úscatter_meta_implb  s
    rµ  c                 C   s   t | |||dƒ |  | j¡S ©Nr¦  ©rµ  rm   rn   ©ro   r_   ri   rÂ   r)   r)   r*   Úmeta_scatter_addk  s    r¹  c                 C   s   t | |||dƒ | S r¶  ©rµ  r¸  r)   r)   r*   Úmeta_scatter_add_q  s    r»  c                 C   s0   t |tjƒr|nd }t| ||||ƒ |  | j¡S r$   )rN   r=   r
   rµ  rm   rn   ©ro   r_   ri   Zsrc_or_valuerì   rÂ   r)   r)   r*   Úmeta_scatterw  s    
r½  c                 C   s(   t |tjƒr|nd }t| ||||ƒ | S r$   )rN   r=   r
   rµ  r¼  r)   r)   r*   Úmeta_scatter_†  s    	r¾  )r¡  Úqueryr�   r;  r�   Ú	logsumexpÚ	cum_seq_qÚ	cum_seq_kÚmax_qÚmax_kÚ	dropout_pÚ	is_causalÚphilox_seedÚphilox_offsetr   c                 C   sX   t  | dd¡¡ dd¡}t  | dd¡¡ dd¡}t  | dd¡¡ dd¡}|||fS rô  )r=   r€   r[  )r¡  r¿  r�   r;  r�   rÀ  rÁ  rÂ  rÃ  rÄ  rÅ  rÆ  rÇ  rÈ  r   Úgrad_qÚgrad_kÚgrad_vr)   r)   r*   Ú'meta__scaled_dot_product_flash_backward”  s    rÌ  ç        )r¿  r�   r;  rÅ  rÆ  Ú	attn_maskr   c                 C   sv   |   d¡}|   d¡}|   d¡}	|   d¡}
tj||	||
f| j| jd� dd¡}tj||	|ftj| jd� dd¡}||fS )Nr   r’   r­   rs   re  )rv   r=   rd   rD   r\   r[  r@   )r¿  r�   r;  rÅ  rÆ  rÎ  r   r¦   Ú	num_headsZmax_seqlen_batch_qÚhead_dimZ	attentionrÀ  r)   r)   r*   Ú0meta__scaled_dot_product_flash_attention_for_cpu°  s4    




ý üýù ø
þrÑ  )
r¡  r¿  r�   r;  r�   rÀ  rÅ  rÆ  rÎ  r   c
                 C   s–   |  d¡}
|  d¡}|  d¡}|  d¡}|  d¡}tj|
|||fd|j|jd�}tj|
|||fd|j|jd�}tj|
|||fd|j|jd�}|||fS )Nr   r’   rs   r­   ©r   r­   r’   rs   re  )rv   r=   Úempty_permutedrD   r\   )r¡  r¿  r�   r;  r�   rÀ  rÅ  rÆ  rÎ  r   r¦   rÏ  rÐ  Úlen_qZlen_krÉ  rÊ  rË  r)   r)   r*   Ú9meta__scaled_dot_product_flash_attention_for_cpu_backward×  s0    





ü
ü
ürÕ  )r¡  r¿  r�   r;  Ú	attn_biasr�   rÀ  rÇ  rÈ  rÅ  Úgrad_input_maskrÆ  r   c                 C   s  |  d¡}|  d¡}|  d¡}|  d¡}|  d¡}|  d¡}tj||||fd|j|jd�}tj||||fd|j|jd�}tj||||fd|j|jd�}d }|d k	�r|
d �r|  d¡}|d dkrÈ|n|d |d  }t|  ¡ ƒ}||d< tj||j|jd�}|d	d |…f }||||fS )
Nr   r’   r­   rs   rÒ  re  rq   rá   .)rv   r=   rÓ  rD   r\   r•   rd   )r¡  r¿  r�   r;  rÖ  r�   rÀ  rÇ  rÈ  rÅ  r×  rÆ  r   r¦   rÏ  rÃ  rÐ  Z
head_dim_vrÄ  rÉ  rÊ  rË  Ú	grad_biasÚlastDimÚlastDimAlignedÚ	new_sizesr)   r)   r*   Ú+meta__scaled_dot_product_efficient_backward  sJ    






ü
ü
ü
   ÿrÜ  )r¡  r¿  r�   r;  r�   rÀ  rÁ  rÂ  rÃ  rÄ  rÅ  rÆ  rÇ  rÈ  r   Úwindow_size_leftÚwindow_size_rightc                 C   s(   t  |¡}t  |¡}t  |¡}|||fS r$   râ  )r¡  r¿  r�   r;  r�   rÀ  rÁ  rÂ  rÃ  rÄ  rÅ  rÆ  rÇ  rÈ  r   rÝ  rÞ  Ú
grad_queryÚgrad_keyÚ
grad_valuer)   r)   r*   Úmeta__flash_attention_backwardB  s    


râ  )r¡  r¿  r�   r;  rÍ   Úcu_seqlens_qÚcu_seqlens_kÚmax_seqlen_qÚmax_seqlen_krÀ  rÅ  rÇ  rÈ  Úcustom_mask_typeÚbias_requires_gradr   Únum_splits_keyÚshared_storage_dqdkdvc                 C   sF  |ržt  |jd |jd kdd„ ¡ t  |jd |jd kdd„ ¡ t j|jdd… d|jd |jd f˜|j|jd	�}| d
d¡}| d
d¡}| d
d¡}nt  |¡}t  |¡}t  |¡}|d k	�r*| d¡}|d dkrà|n|d |d  }t	| ¡ ƒ}||d< t j||j|jd	�}|dd |…f }nt jd|jd�}||||fS )Nr’   c                   S   s   dS )Nz,seqlen must match for `shared_storage_dqdkdvr)   r)   r)   r)   r*   rH   }  rI   z4meta__efficient_attention_backward.<locals>.<lambda>rs   c                   S   s   dS )Nz3embedding dim must match for `shared_storage_dqdkdvr)   r)   r)   r)   r*   rH   �  rI   r   r-  rq   re  rt  r­   rá   .r)   r  )
r=   rK   rn   rd   rD   r\   rÁ  r€   rv   r•   )r¡  r¿  r�   r;  rÍ   rã  rä  rå  ræ  rÀ  rÅ  rÇ  rÈ  rç  rè  r   ré  rê  Úchunkrß  rà  rá  rÙ  rÚ  rÛ  rØ  r)   r)   r*   Ú"meta__efficient_attention_backwarda  s:    þþ"ý




 rì  )ro   rÖ   rÍ   rÏ   Úscale_aÚscale_bÚscale_resultÚuse_fast_accumc                    s8  dd„ }dd„ }	dd„ }
t  ˆ ¡ dko2ˆ  ¡ dk‡ ‡fdd	„¡ t  |ˆ ¡ ƒd
d	„ ¡ t  |	ˆ jˆ  ¡ ƒdd	„ ¡ t  ˆ d¡d dk‡fdd	„¡ t  ˆ  d¡d dko¾ˆ  d¡d dk‡ fdd	„¡ t  |
ˆjƒoâ|
ˆ jƒ‡ ‡fdd	„¡ |d k	rþ|nˆj}t jˆ d¡ˆ  d¡|ˆjd�t jdt j	ˆjd�fS )Nc                 S   s   | d | d ko| d dkS r  r)   r  r)   r)   r*   Úis_row_major¨  s    z$meta_scaled_mm.<locals>.is_row_majorc                 S   s   |d dko|d | d kS r  r)   )rn   r˜   r)   r)   r*   Úis_col_major«  s    z$meta_scaled_mm.<locals>.is_col_majorc                 S   s   | t jt jt jt jfkS r$   )r=   Zfloat8_e4m3fnZfloat8_e5m2Zfloat8_e4m3fnuzZfloat8_e5m2fnuzrg   r)   r)   r*   Úis_fp8_type®  s    üz#meta_scaled_mm.<locals>.is_fp8_typer­   c                      s   dˆ  ¡ › dˆ   ¡ › �S )Nz%Inputs must be 2D but got self.dim()=z and mat2.dim()=rr   r)   ©rÖ   ro   r)   r*   rH   ¸  rI   z meta_scaled_mm.<locals>.<lambda>c                   S   s   dS )Nzself must be row_majorr)   r)   r)   r)   r*   rH   ¼  rI   c                   S   s   dS )Nzmat2 must be col_majorr)   r)   r)   r)   r*   rH   À  rI   r’   rá   r   c                      s   dˆ   d¡› �S )NzBExpected self.size(0) to be divisible by 16, but got self.size(1)=r’   ru   r)   r‚   r)   r*   rH   Ä  rI   c                      s   dˆ j › �S )Nz>Expected both dimensions of mat2 to be divisble by 16 but got rz  r)   )rÖ   r)   r*   rH   È  rI   c                      s   dˆj › dˆ j › �S )Nz8Expected both inputs to be fp8 types but got self.dtype=z and mat2.dtype=rg   r)   rô  r)   r*   rH   Ì  rI   re  r)   )
r=   rK   r_   r˜   rn   rv   rD   rd   r\   râ   )ro   rÖ   rÍ   rÏ   rí  rî  rï  rð  rñ  rò  ró  Z
_out_dtyper)   rô  r*   Úmeta_scaled_mm�  sH    þ
þþ
þ"
þþ   ÿþrõ  c                 C   s    t | ||||dd� |  | j¡S ©NT)r©  r·  ©ro   r_   ri   rÂ   rì   rê   r)   r)   r*   Úmeta_scatter_reduce_twoÔ  s    rø  c                 C   s   t | ||||dd� | S rö  rº  r÷  r)   r)   r*   Úmeta_scatter_reduce__twoÛ  s    rù  c                   sh   t  dˆ  ¡   k odkn  ‡ fdd„¡ ˆ  ¡ dkrLt j|t jˆ jd�S t jˆ  d¡|t jˆ jd�S )Nr   r­   c                      s   dˆ   ¡ › �S )Nz@The probabilty distributions dimensions must be 1 or 2, but got rr   r)   rX  r)   r*   rH   æ  rI   z"meta_multinomial.<locals>.<lambda>r’   re  )r=   rK   r_   rd   rj   r\   rv   )r¥   Únum_samplesÚreplacementr±   r)   rX  r*   Úmeta_multinomialá  s    
þ   ÿrü  c                 C   s   d}| D ]}||9 }q|S rÄ   r)   )Úvsr�  Úvr)   r)   r*   Úmultiply_integersï  s    
rÿ  c                    sž   t  tˆƒˆk‡‡fdd„¡ ˆd ‰ t  tˆƒˆ k‡ ‡fdd„¡ t  tdd„ ˆdd … D ƒƒortdd„ ˆD ƒƒ‡‡fdd„¡ ˆd d… \}}||fˆ˜S )Nc                      s   dˆ › dt ˆƒ› �S )Nz%It is expected output_size equals to ú, but got size rÆ  r)   )Únum_spatial_dimsrœ  r)   r*   rH   ù  rI   z'upsample_common_check.<locals>.<lambda>r­   c                      s   dˆ › dt ˆƒ› �S )Nz$It is expected input_size equals to r   rÆ  r)   )Úexpected_input_dimsr…  r)   r*   rH   þ  rI   c                 s   s   | ]}|d kV  qdS ©r   Nr)   )r3   r  r)   r)   r*   rQ     s     z(upsample_common_check.<locals>.<genexpr>c                      s   dˆ › dˆ› �S )NzDInput and output sizes should be greater than 0, but got input size z and output size r)   r)   )r…  rœ  r)   r*   rH     rI   )r=   rK   r”   r®  )r…  rœ  r  rÔ  Zchannelsr)   )r  r…  r  rœ  r*   Úupsample_common_checkö  s    
þ
þ*þr  c                    sZ   t  ˆ  ¡ dkp"tˆ  ¡ dd … ƒ‡ fdd„¡ tˆ  ¡ |dd�}ˆ  |¡jt 	ˆ ¡d�S )Nr   r’   c                      s   dˆ   ¡ › �S )Nz>Non-empty 3D data tensor expected but got a tensor with sizes ru   r)   rX  r)   r*   rH     rI   z$upsample_nearest1d.<locals>.<lambda>©r  r|   ©
r=   rK   rl   rÿ  rv   r  rm   rÀ   r8   r:  )r¥   rœ  ZscalesÚfull_output_sizer)   rX  r*   Úupsample_nearest1d  s    
þ  ÿ
ÿr  c           	         sŽ   t  ˆ  ¡ dkp"tˆ  ¡ dd … ƒ‡ fdd„¡ tˆ  ¡ |dd�}ˆ  |¡}t ˆ ¡}ˆ j	\}}}}ˆ j
jdkr~|dk r~t j}|j|d	�}|S )
Nr   r’   c                      s   dˆ   ¡ › �S ©Nz>Non-empty 4D data tensor expected but got a tensor with sizes ru   r)   rX  r)   r*   rH   !  rI   z$upsample_nearest2d.<locals>.<lambda>r­   r  r�  rè  r|   )r=   rK   rl   rÿ  rv   r  rm   r8   r:  rn   r\   rU   r�   rŠ   )	r¥   rœ  Úscales_hÚscales_wr  r«   r}   r;   Z
n_channelsr)   rX  r*   Úupsample_nearest2d  s     
þ  ÿ

r  ©rá  rœ  r…  r
  r  c                    st   t ||dd�‰ t ˆjdk‡fdd„¡ tdƒD ]*‰t ˆ ˆ¡ˆ ˆ k‡ ‡‡fdd„¡ q0ˆ |¡jt 	ˆ¡d�S )Nr­   r  rè  c                      s   dˆ j › �S )NzFExpected grad_output to be a tensor of dimension 4 but got: dimension r_  r)   r¯  r)   r*   rH   G  rI   z-upsample_nearest2d_backward.<locals>.<lambda>c                
      s&   dˆ› dˆ ˆ › dˆ› dˆ  ˆ¡› �S )NzCExpected grad_output to have the same shape as output; output.size(z) = z but got grad_output.size(ru   r)   ©r  rá  r§   r)   r*   rH   L  s    r|   )
r  r=   rK   ry   r–   rv   rm   rÀ   r8   r:  r  r)   r  r*   Úupsample_nearest2d_backward5  s"      ÿ
þþ	
ÿr  c                    sZ   t  ˆ  ¡ dkp"tˆ  ¡ dd … ƒ‡ fdd„¡ tˆ  ¡ |dd�}ˆ  |¡jt 	ˆ ¡d�S )Nr   r’   c                      s   dˆ   ¡ › �S )Nz>Non-empty 5D data tensor expected but got a tensor with sizes ru   r)   rX  r)   r*   rH   ^  rI   z$upsample_nearest3d.<locals>.<lambda>rs   r  r|   r  )r¥   rœ  Zscales_dr
  r  r  r)   rX  r*   Úupsample_nearest3dX  s    
þ  ÿ
ÿr  c           
      C   sª   t  | ¡t j| t jd� }}|d k	r¢|d k	r¢t|tƒs:t‚t|tƒsHt‚|j}| ¡ }	t||ƒ}t||ƒ}| 	||	¡ | 	||	¡ t
||d� t
||d� ||fS ||fS )Nrg   )r«  r¬  )r=   r€   r‡   rN   r   rb   rn   r˜   r   rÉ   r   )
ro   Ústabler_   Ú
descendingr„   r…   rþ  r§   rz   Z
out_strider)   r)   r*   Ú	meta_sorth  s    	

r  )r_   r  c                C   s   t | |||d�d S )N)r  r_   r  r’   )r  )ro   r  r_   r  r)   r)   r*   Úmeta_argsortƒ  s    r  c                    s  t  ˆjdk‡fdd„¡ t  ˆjˆjk‡‡fdd„¡ ˆ d¡‰ˆd k	r t  ˆjdk‡fdd„¡ t  ˆ ¡ ˆk‡‡fdd„¡ t  ˆjˆjk‡‡fdd„¡ t  ˆjdk‡fd	d„¡ ˆ d
¡ˆ ˆ ‰ t  ˆ ¡ ˆ k‡ ‡‡‡‡fdd„¡ t  t‡fdd„ˆˆˆˆfD ƒƒdd„ ¡ d S )Nr­   c                      s   ˆ j › d�S ©Nz != 2r_  r)   ©Úinput_gatesr)   r*   rH   ‹  rI   z%rnn_cell_checkSizes.<locals>.<lambda>c                      s   ˆj › dˆ j › �S ©Nú != rz  r)   )Úhidden_gatesr  r)   r*   rH   Ž  rI   r’   c                      s   ˆ j › d�S )Nz != 1r_  r)   )Ú
input_biasr)   r*   rH   ’  rI   c                      s   ˆ  ¡ › dˆ › �S r  r  r)   )Ú
gates_sizer  r)   r*   rH   •  rI   c                      s   ˆj › dˆ j › �S r  rz  r)   )Úhidden_biasr  r)   r*   rH   ™  rI   c                      s   ˆ j › d�S r  r_  r)   )Úprev_hiddenr)   r*   rH   ›  rI   r   c                
      s,   ˆ  ¡ › dˆ d¡› dˆ› dˆ› dˆ › d�
S )Nr  r   z * z // z (aka rT   )rl   rv   r)   )Úexpected_prev_hidden_numelÚfactorr  r  r  r)   r*   rH   Ÿ  rI   c                 3   s   | ]}|j ˆ j kV  qd S r$   r  r2   r  r)   r*   rQ   ¢  s   ÿz&rnn_cell_checkSizes.<locals>.<genexpr>c                   S   s   dS )Nz%expected all inputs to be same devicer)   r)   r)   r)   r*   rH   ¦  rI   )r=   rK   ry   rn   rv   rl   r®  )r  r  r  r  r   r  r)   )r  r   r  r  r  r  r  r  r*   Úrnn_cell_checkSizesˆ  s8    
þ

þ
þ
þ
þûr!  c                 C   sL   t | |||d|ƒ tj| tjd�}tj|tjd�}tj|tjd�}|||fS )Nrè  r|   )r!  r=   r€   r�   )r  r  Úcxr  r  Ú	workspaceÚhyÚcyr)   r)   r*   Ú_thnn_fused_lstm_cell_metaª  s
    r&  c                 C   s*  t |ƒdk}|r,t |ƒ}|d }| jd }n4|
r:| jd n| jd }|
rR| jd n| jd }d}|rhdnd}|dkrx|n|}|rŽ||| g}n |
r |||| gn|||| g}|  |¡}|	| ||g}|d kràtjd| jd�}n
| |¡}| |	| ||g¡}|�rdnd}| j|tjd�}|||||fS )Nr   r’   rq   r­   r  rg   )r”   rn   rm   r=   rd   r\   r>  )r¥   rË   Zweight_stride0Z
weight_bufÚhxr"  r�  Úhidden_sizeZ	proj_sizeÚ
num_layersÚbatch_firstZdropoutÚtrainÚbidirectionalÚbatch_sizesZdropout_stateZis_input_packedÚ
seq_lengthÚ
mini_batchZbatch_sizes_sumZnum_directionsZout_sizerz   r«   Z
cell_shaper%  r$  Zreserve_shapeZreserver)   r)   r*   Ú
_cudnn_rnnµ  s2    ÿý

r0  c                 C   sÂ   |r| j d n| j d }|r&| j d n| j d }|
}|rB|||gn|||g}|  |¡}|d krptjd| jd�}n| |j ¡}|d kr–tjd| jd�}n| |j ¡}tjd| jtjd�}||||fS )Nr’   r   r  r†   )rn   rm   r=   rd   r\   r>  )r¥   Zw0Zw1Zw2Zw3Úhx_Zcx_r‘   r-  r�  r(  r)  Ú
has_biasesr,  r*  r+  r.  r/  Zoutput_chanelsrz   r«   r$  r%  r#  r)   r)   r*   Úmkldnn_rnn_layerí  s     ÿý
r3  c                    sR   | j dkr.t ˆ dkpˆ dk‡ ‡fdd„¡ n t |  ˆ ¡dk‡ ‡fdd„¡ d S )Nr   rq   c                      s   ˆ› dˆ › �S )Nz4: Expected reduction dim -1 or 0 for scalar but got r)   r)   ©r_   rE  r)   r*   rH     rI   z'zero_numel_check_dims.<locals>.<lambda>c                      s   ˆ› dˆ › d�S )Nz: Expected reduction dim z to have non-zero size.r)   r)   r4  r)   r*   rH     rI   )ry   r=   rk   rv   )ro   r_   rE  r)   r4  r*   Úzero_numel_check_dims  s    
þþr5  c                    sD   |d k	r$t || ¡ ƒ}t||ˆ ƒ nt | ¡ dk‡ fdd„¡ d S )Nr   c                      s
   ˆ › d�S )Nz@: Expected reduction dim to be specified for input.numel() == 0.r)   r)   rk  r)   r*   rH   *  rI   z%check_argmax_argmin.<locals>.<lambda>)rˆ   r_   r5  r=   rK   rl   )r2  ro   r_   r)   rk  r*   Úcheck_argmax_argmin#  s    

þr6  c                 C   sD   t d| |ƒ t | j|d k	r"|fnd ¡}t| ||ƒ}| j|tjd�S )NÚargmaxrg   )r6  r8   r  rn   r  rm   r=   r‡   )ro   r_   r  r+  rn   r)   r)   r*   Úargmax_argmin_meta.  s    r8  c                 C   s   t jd||||d�S )Nr)   rµ   r·   )r  rD   r[   r\   r]   r)   r)   r*   Úscalar_tensor6  s        ÿr9  c                 C   s²   t ||  ¡ dd�}t |dko:||  ¡ dkr6|  |¡ndkdd„ ¡ |  ¡ dkrTdn|  |¡}t |dkop||kdd„ ¡ t| jƒ}t|ƒdkr˜|||< |  |¡| j|tj	d�fS )	NT)rš  r   r’   c                   S   s   dS )Nzselected index k out of ranger)   r)   r)   r)   r*   rH   C  rI   ztopk_meta.<locals>.<lambda>c                   S   s   dS )Nzk not in range for dimensionr)   r)   r)   r)   r*   rH   F  rI   rg   )
rˆ   r_   r=   rK   rv   r•   rn   r”   rm   r‡   )ro   rå   r_   ÚlargestÚsortedZ	sliceSizeZtopKSizer)   r)   r*   Ú	topk_meta=  s    $þ
r<  c                 C   sä   | d k	r| n|}t  | ¡ dkdd„ ¡ | ¡ }| d k	rPt  |  ¡ |kdd„ ¡ |d k	rpt  | ¡ |kdd„ ¡ t  | ¡ |kdd„ ¡ t  | ¡ |kdd„ ¡ t  | ¡ dkdd„ ¡ t  | ¡ |d	 |d
  d kdd„ ¡ d S )Nr­   c                   S   s   dS ©NÚ r)   r)   r)   r)   r*   rH   T  rI   z(checkLSTMBackwardSizes.<locals>.<lambda>c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   W  rI   c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   Y  rI   c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   Z  rI   c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   [  rI   c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   \  rI   r   r’   rè  c                   S   s   dS r=  r)   r)   r)   r)   r*   rH   ]  rI   )r=   rK   r_   rv   rl   )Úgrad_hyÚgrad_cyr"  r%  r#  Zdefined_gradZexp_sizer)   r)   r*   ÚcheckLSTMBackwardSizesR  s    rA  c           	      C   s`   | d kr|d krdS t | ||||ƒ tj|td�}tj|td�}|rR|jddd�nd }|||fS )N)NNNr|   r   F)r  )rA  r=   r€   Úlegacy_contiguous_memory_formatr¥  )	r?  r@  r"  r%  r#  Zhas_biasZ
grad_gatesZgrad_cxrØ  r)   r)   r*   Ú#_thnn_fused_lstm_cell_backward_impla  s     ÿrC  c                 C   sf   d }d }d }|d r"|  |  ¡ ¡}|d s2|d r\|  | d¡|  d¡f¡}|  | d¡¡}|||fS )Nr   r’   r­   rq   rÆ  )rÈ  rÇ  rÉ  rÊ  rã  Zgrad_weightrØ  r)   r)   r*   Úlinear_backwardo  s    rD  c                    s²   t ˆjƒdkr$ˆjd ||  dks:tdˆj› d|› �ƒ‚dd„ ‰ ‡ ‡fdd	„}ˆjd ||  }ˆjd
 | }ˆjd | }ˆjd d… |||f˜}ˆ |¡}|j|ƒ d�}|S )Nr­   rt  r   z'Invalid input shape for pixel_shuffle: z with upscale_factor = c                 S   s   t j | ¡t jkS r$   r8  r<  r)   r)   r*   r=  ‚  s    z,meta_pixel_shuffle.<locals>.is_channels_lastc                      sN   ˆ ˆƒr"t ˆƒdkrtjS tjS n(ˆjtjd�r6tjS ˆjtjd�rJtjS d S r>  )rž  r=   r�   r;  rµ  r?  r)   ©r=  ro   r)   r*   rA  …  s    z.meta_pixel_shuffle.<locals>.pick_memory_formatr-  rq   r|   )r”   rn   rb   rm   rÀ   )ro   Zupscale_factorrA  r‹  ZHrZWrrz   r�   r)   rE  r*   Úmeta_pixel_shuffle|  s    ÿÿþ
rF  c                 C   sZ   |   | j¡}|  |j¡}|  |j¡}|  |j¡}|  |j¡}|  |j¡}|||||||fS r$   rK  )r¥   Zweight0Zweight1Zweight2Zweight3r1  Zcx_tmpr«   Zhy_Zcy_Zgrad_output_r_optZgrad_hy_r_optZgrad_cy_r_optr‘   r�  r(  r)  r2  r+  r,  r-  r*  r#  Zdiff_xZdiff_hxZdiff_cxZdiff_w1Zdiff_w2Zdiff_br)   r)   r*   Úmkldnn_rnn_layer_backwardš  s    rG  )Ú	out_int32r£   c                C   s   t j| |rt jnt jd� ¡ S r  )r=   r€   rÒ   r‡   rŠ   )ro   Z
boundariesrH  r£   r)   r)   r*   Úmeta_bucketize½  s     ÿrI  éd   c                    s¾   d‰t ˆƒdkr(t ˆ ¡ ‡fdd„¡ t tˆ tƒ‡ ‡fdd„¡ t ˆ dk‡ ‡fdd„¡ t tˆtƒ‡‡fdd„¡ t tˆtƒ‡‡fd	d„¡ t ˆˆkd
d„ ¡ tjˆ ˆjˆj	d�S )Nzhistc()r  c                      s   dˆ j › d�S )Nz%"histogram_cpu" not implemented for 'ú'rg   r)   rX  r)   r*   rH   Ì  rI   zmeta_histc.<locals>.<lambda>c                      s   ˆ› dt ˆ ƒ› �S )Nz#: argument 'bins' must be int, not rÑ  r)   ©ÚbinsrE  r)   r*   rH   Ð  rI   r   c                      s   ˆ› dˆ › �S )Nz: bins must be > 0, but got r)   r)   rL  r)   r*   rH   Ò  rI   c                      s   ˆ › dt ˆƒ› �S )Nz%: argument 'min' must be Number, not rÑ  r)   )rE  r  r)   r*   rH   Õ  rI   c                      s   ˆ › dt ˆƒ› �S )Nz%: argument 'max' must be Number, not rÑ  r)   )rE  r  r)   r*   rH   Ù  rI   c                   S   s   dS )Nz&{fn_name}: max must be larger than minr)   r)   r)   r)   r*   rH   Û  rI   r†   )
rž  r=   rK   r®   rN   r   r   rd   r\   rD   )r¥   rM  r  r  r)   )rM  rE  r¥   r  r  r*   Ú
meta_histcÅ  s*    
þþþþrN  c                    sd   t ˆ  ¡ |dd�}t ˆ  ¡ dkp>tdd„ ˆ  ¡ dd … D ƒƒ‡ fdd„¡ ˆ  |¡jt 	ˆ ¡d	�S )
Nr­   r  r   c                 s   s   | ]}|d kV  qdS r  r)   )r3   rv   r)   r)   r*   rQ   é  s     z,meta_upsample_bimode2d_aa.<locals>.<genexpr>r’   c                      s   dˆ   ¡ › �S r	  ru   r)   rX  r)   r*   rH   ê  rI   z+meta_upsample_bimode2d_aa.<locals>.<lambda>r|   )
r  rv   r=   rK   rl   r®  rm   rÀ   r8   r:  )r¥   rœ  rˆ  r
  r  r  r)   rX  r*   Úmeta_upsample_bimode2d_aaß  s      ÿ(
þ
ÿrO  c                 C   s\   t  | ¡ dkdd„ ¡ t  | ¡ dkdd„ ¡ t  |jjdd„ ¡ t  |jjdd„ ¡ d S )Nr’   c                   S   s   dS )Nz%found_inf must be a 1-element tensor.r)   r)   r)   r)   r*   rH   õ  rI   z<_amp_foreach_non_finite_check_and_unscale_.<locals>.<lambda>c                   S   s   dS )Nz%inv_scale must be a 1-element tensor.r)   r)   r)   r)   r*   rH   ø  rI   c                   S   s   dS )Nz!found_inf must be a float tensor.r)   r)   r)   r)   r*   rH   ü  rI   c                   S   s   dS )Nz!inv_scale must be a float tensor.r)   r)   r)   r)   r*   rH      rI   )r=   rK   rl   rD   r®   )ro   rÏ  Z	inv_scaler)   r)   r*   Ú*_amp_foreach_non_finite_check_and_unscale_ò  s     
 ÿ
 ÿþþrP  c                 C   s   t |  ¡ ƒ}|  |¡S r$   )r•   rv   rm   )ro   ÚnanZposinfZneginfrò   r)   r)   r*   Ú
nan_to_num  s    rR  c                 C   s®   | j tjtjtjtjhks,td| j › d�ƒ‚| j}t||ƒ}t||ƒ}||krR| S t	|  
¡ ƒ}t	|  ¡ ƒ}|| ||  ||< ||< || ||  ||< ||< |  ||¡ | S )Nz>torch.transpose_: in-place transposition is not supported for z layout)r[   r=   r¼  Z
sparse_cscr½  Z
sparse_bscrb   ry   rˆ   r•   rv   r˜   rÉ   )ro   Zdim0r  Úndimsrv   r˜   r)   r)   r*   r¶    s&    üû

r¶  c                 C   sx   | j }| jrD|  ¡ }|  ¡ }|dkr,|dks`td|› d|› d�ƒ‚n|  ¡ dks`td|› d�ƒ‚t| d|dk rrdndƒS )	Nr­   r   zEt_ expects a tensor with <= 2 sparse and 0 dense dimensions, but got z sparse and z dense dimensionsz6t_ expects a tensor with <= 2 dimensions, but self is rñ  r’   )ry   r�  r�  r‘  rb   r_   r¶  )ro   rS  r�  r‘  r)   r)   r*   Út_'  s    ÿÿþ
ÿ
þrT  )rH  r£   ÚsideÚsorterc                C   sD   |r
t jnt j}t|t jƒr.t j||d� ¡ S t jd|| jd�S d S )Nrg   r)   re  )	r=   rÒ   r‡   rN   r
   r€   rŠ   rd   r\   )Zsorted_sequencero   rH  r£   rU  rV  rD   r)   r)   r*   Úmeta_searchsorted9  s    rW  c                    s,   t  ˆ t jt jt jt jfk‡ fdd„¡ d S )Nc                      s
   dˆ › �S )Nz/Unsupported input type encountered for isin(): r)   r)   rg   r)   r*   rH   H  rI   z3_check_for_unsupported_isin_dtype.<locals>.<lambda>)r=   rK   r  rä   Z
complex128Z	complex64rg   r)   rg   r*   Ú!_check_for_unsupported_isin_dtypeE  s    
þrX  c
                    sf   t  ˆ jt jt jt jt jfk‡ fdd„¡ tdƒ\}
}}||krNt  |d k	¡ ˆ  |ˆ  	d¡f¡}|S )Nc                      s   dˆ j › �S )Nz$Unsupported input type encountered: rg   r)   ©r  r)   r*   rH   [  rI   z3meta_embedding_bag_dense_backward.<locals>.<lambda>rs   r’   )
r=   rK   rD   rã   rä   râ   Zfloat64r–   rm   rv   )r  r…   r  r  Zmaximum_indicesZnum_weightsr  r�  r  r
  r  r  r  Zindex_grad_weightr)   rY  r*   Ú!meta_embedding_bag_dense_backwardL  s    
þrZ  c                 C   sŒ   t dƒ\}}}	|  d¡}
t ||kd¡ t |  ¡ dk¡ t | ¡ dk¡ | d¡}t | ¡ dk¡ t | d¡|
k¡ |  |f¡}|S )Nrs   r’   zHembedding_bag_backward: per_sample_weights only supported for mode='sum'r­   r   )r–   rv   r=   rK   r_   rm   )r  rË   r…   rõ   r  r�  r
  r  r  r  Zembedding_featuresrú  r«   r)   r)   r*   Ú.meta_embedding_bag_per_sample_weights_backwardd  s    
þ
r[  )Úassume_uniqueÚinvertc                C   sx   t  t| tƒpt|tƒdd„ ¡ t| tƒs:t j| |jd�} t|tƒsTt j|| jd�}t| jƒ t|jƒ t j| t j	d�S )Nc                   S   s   dS )Nz<At least one of elements and test_elements must be a Tensor.r)   r)   r)   r)   r*   rH   |  rI   zmeta_isin.<locals>.<lambda>r  rg   )
r=   rK   rN   r
   rÆ   r\   rX  rD   r€   r  )ÚelementsZtest_elementsr\  r]  r)   r)   r*   Ú	meta_isinw  s    þ



r_  )r³   ro   rí   c                 C   s4   t  | dkdd„ ¡ t|tjd�\}}t j||d�S )Nr   c                   S   s   dS )Nz,polygamma(n, x) does not support negative n.r)   r)   r)   r)   r*   rH   Œ  rI   z meta_polygamma.<locals>.<lambda>r  rg   )r=   rK   r   r   r  r€   )r³   ro   r;   r6   r)   r)   r*   Úmeta_polygamma‰  s    þ
r`  c                 C   s6   |   ¡ �^ }}}}tj||||fž| j| j| jdœŽS )N)rD   r[   r\   )rv   r=   rd   rD   r[   r\   )r¥   r1  Zleading_dimsr‹  rn  ro  r)   r)   r*   Úmeta_channel_shuffle”  s    üùra  r‚   c                 C   s   t dƒ‚d S )Nz.Tensor.item() cannot be called on meta tensors)r¿   r‚   r)   r)   r*   Úmeta_local_scalar_dense¤  s    rb  c                 C   s   t | ƒtƒ dd„ ƒƒ}|S )Nc                 S   s   t | tjd�S r(  ©r<   r   r  r�   r)   r)   r*   Ú_fª  s     ÿz)_create_unary_float_meta_func.<locals>._f©r.   r   ©Úfuncrd  r)   r)   r*   Ú_create_unary_float_meta_func©  s    rh  c                 C   s   t | ƒtƒ dd„ ƒƒ}|S )Nc                 S   s   t | |tjd�S r(  rc  )r4   rH  r)   r)   r*   rd  µ  s
      ÿz*_create_binary_float_meta_func.<locals>._fre  rf  r)   r)   r*   Ú_create_binary_float_meta_func´  s    ri  c                  C   s8  i } dD ]*}t | }|D ]}|| kr|| | |< qq|  ¡ D ]ö\}}t|tjjƒrTq<t|tƒsbt‚| tj	j
j¡|ƒ tj	 | ¡ d¡r¤|t d kr¢t|› d�ƒ‚q<|jr¬q<| ¡ dkrºq<d| ¡ krÔt ||¡ q<d| ¡ krît ||¡ q<d| ¡ k�r
t ||¡ q<d	| ¡ k�r&t ||¡ q<t ||¡ q<d S )
N)rZ   Zpost_autogradZpre_autogradZCompositeImplicitAutogradrZ   z« is a CompositeImplicitAutograd op, we shouldn't register meta function for it. Instead, we should let the decomposition run and write meta kernels for the base operators.>   úaten::copy_úaten::cloneúaten::as_strided_scatterúaten::empty_stridedúaten::rot90úaten::constant_pad_ndúaten::_to_copyzmkldnn::zmkl::zonednn::zquantized::)r   ÚitemsrN   r=   Z_opsZHigherOrderOperatorr   rb   Zpy_implÚ_CZDispatchKeyr#   Z%_dispatch_has_kernel_for_dispatch_keyr2  r¿   Zis_viewÚ2_meta_lib_dont_use_me_use_register_meta_for_mkldnnÚimplÚ/_meta_lib_dont_use_me_use_register_meta_for_mklÚ2_meta_lib_dont_use_me_use_register_meta_for_onednnÚ5_meta_lib_dont_use_me_use_register_meta_for_quantizedÚ'_meta_lib_dont_use_me_use_register_meta)Zactivate_meta_tablerU   ÚregistryZopoZop_overloadr(   r)   r)   r*   Úactivate_metaß  sH     ÿÿ	 ÿrz  )F)NNN)N)NNNF)F)F)NN)NNN)NN)T)r1  )rè   )rL  T)F)F)F)FF)F)TT)rŽ  )FTN)TFF)TF)r­   )r  N)r  N)FF)N)r)   r“   r[  F)r)   r“   FTN)r)   r“   FTN)N)Fr   FNFrq   )T)NF)rq   F)r’   )F)F)N)F)r)   r“   r[  F)r)   r“   r[  F)NNNNN)r   NNr’   )T)F)F)N)N)NNF)N)N)N)rÍ  FNN)NN)FN)NNN)NNF)NNNNNF)T)T)F)N)NN)NN)NNN)Nrq   FNN)NN)NF)NNNN)rq   TT)rJ  r   r   )NN)NNN)rq   )rq   (3  rÿ  Úenumr   Útypingr   r   r   r   r   r=   Ztorch._prims_commonr9  r8   r   r	   r
   Ztorch._decompr   r   r   r   Z
torch._opsr   Ztorch._primsr   r   r   r   r   r   r   r   r   r   Ztorch._prims_common.wrappersr   r   r   r   r   rÂ  r   r    Ztorch.utilsr!   r,   Úopsr"   ÚlibraryÚLibraryrx  r.   r<   rE   rL   ZlinspaceZlogspacer»  rf   ZtakerÁ   r�   rp   r{   rƒ   ZcummaxZcumminr‰   r‹   r¨   Z_fft_c2cr¬   Z_fft_r2cr°   ZrandpermZgenerator_outr´   rj   r¸   Úrandintrº   r»   Zlow_outr¼   Zrandr½   Z_fft_c2rr¾   r  rÃ   rÈ   Z
unsqueeze_rÊ   Z_sparse_semi_structured_linearr-  rD   rÓ   Z_sparse_semi_structured_mmrØ   Z_sparse_semi_structured_addmmrÛ   Z_cslt_sparse_mmr  ré   Zindex_reducerw  rð   Zindex_reduce_rñ   Zindex_selectró   Zsegment_reducerÿ   r  Z	unary_outr  r_   r	  r  r
  r  r  r  r  Z_assert_asyncr  Úmsgr  Z_printr  Z_make_dep_tokenr  r"  Z_functional_sym_constrain_ranger&  r(  Z(_functional_sym_constrain_range_for_sizer)  Z_functional_assert_asyncr*  r~   r7  r   r?  rC  rH  rN  Z_linalg_eighrO  rT  Z_linalg_eigvalsZlinalg_eigvalsrW  Z
linalg_eigrY  r\  r]  r`  rb  re  rh  rp  Zlinalg_inv_exrr  Zlinalg_ldl_factor_exrx  Zlinalg_ldl_solver  Z	linalg_lurƒ  Zlinalg_lu_factor_exr…  Zlinalg_lu_solverˆ  Z	lu_unpackrŒ  r“  Z	linalg_qrr–  r™  r—  Z_linalg_svdr¡  r}  ra  r©  r³  Zlinalg_solve_triangularr·  rº  r¾  Z_linalg_detr¿  rÅ  rÈ  rÖ  Zreflection_pad1drÛ  Zreplication_pad1drÝ  râ  Zreflection_pad1d_backwardræ  Zreplication_pad1d_backwardrç  rò  Zreflection_pad2drô  Zreplication_pad2drõ  Zreflection_pad2d_backwardrã  Zreplication_pad2d_backwardrø  r   Zreflection_pad3dr  Zreplication_pad3dr  Zreflection_pad3d_backwardZreplication_pad3d_backwardr  Z_pdist_forwardr@   r  Z_pdist_backwardr
  Zbaddbmmr  Z	bernoullir  Z
bernoulli_r  r  r  Z_fused_moving_avg_obs_fq_helperr"  Úmmr(  r  rž  r7  r=  ZconvolutionrC  rr  Z_has_mkldnnrs  rD  Z_convolution_pointwiserI  Z_linear_pointwiserL  Zhas_mklru  rM  Z_mkl_linearrN  rv  rO  Zqconv2d_pointwiserX  Zqlinear_pointwiserÆ   rY  rw  rZ  Z
max_pool2drd  rh  Z
avg_pool2dr�  r„  Zavg_pool2d_backwardr†  Z
avg_pool3dr™  Zavg_pool3d_backwardr›  Z_adaptive_avg_pool2dr�  Z_adaptive_avg_pool3drž  Z_adaptive_avg_pool2d_backwardr£  Z_adaptive_avg_pool3d_backwardr¦  r¤  Zadaptive_max_pool2dr®  r°  r²  Zadaptive_max_pool3dr³  r´  rµ  Zrepeat_interleaver·  rO   rº  r¼  ri   Z_unsafe_indexrÅ  Zconvolution_backwardrË  ZaddbmmrÍ  Z_fused_adam_rá  Z_fused_adamrä  Z_int_mmrå  Z_convert_weight_to_int4packrë  Z_weight_int4pack_mmrï  Z_weight_int8pack_mmrð  Z_cdist_forwardrý  Z_cdist_backwardr  Z_embedding_bagr  Z_embedding_bag_forward_onlyr  r  Znansumr  ZmedianZ	nanmedianr  Z
dim_valuesr�  r„   r  Zlogical_not_r   Úrepeatr"  Zzero_r#  Zmul_ZScalarZdiv_Zlogical_and_Zlogical_or_Zlogical_xor_r&  Zadd_Zsub_r'  ÚroundZdecimalsr*  r-  Ú
__rshift__r0  Ú
__lshift__r2  Zzeror3  r’  r5  Úfillr6  Zrelu_r7  Z	index_putZ_unsafe_index_putr:  Zmasked_fill_r<  Z_masked_scaler=  Zmasked_scatter_r?  Zmasked_scatterr@  Zmasked_scatter_backwardrA  Z
index_put_rB  ÚaliasrD  rF  ZbmmrG  rJ  rM  ru  rv  rŽ  r[  rš  r^  Z max_pool2d_with_indices_backwardr]  Zmax_pool2d_with_indicesr^  Zfractional_max_pool2drg  Zmax_unpool2drn  ro  Zmax_unpool3drp  Zmax_pool3d_with_indicesrw  Z max_pool3d_with_indices_backwardrx  r}  r~  r„  Zgrid_sampler_2d_backwardrŠ  rŒ  r�  rŽ  r  rÁ  r”  Zselect_scatterr–  Zslice_scatterr˜  rˆ   r›  r   Zgatherr¤  rª  r®  r¯  r³  rµ  Zscatter_addr¹  Zscatter_add_r»  r´  rÂ   r;  rì   Zvalue_reducer½  Zscatter_r¾  Z,_scaled_dot_product_flash_attention_backwardrÌ  Z+_scaled_dot_product_flash_attention_for_cpurÑ  Z4_scaled_dot_product_flash_attention_for_cpu_backwardrÕ  Z0_scaled_dot_product_efficient_attention_backwardrÜ  Z_flash_attention_backwardrâ  Z_efficient_attention_backwardZSymIntrì  Z
_scaled_mmrõ  Zscatter_reduceZtwoZtwo_outrø  Zscatter_reduce_rù  Zmultinomialrü  rÿ  r  r  Z_upsample_nearest_exact1dr  Z_upsample_nearest_exact2dr  Z"_upsample_nearest_exact2d_backwardr  Z_upsample_nearest_exact3dr™   r  Zvalues_stabler  Zargsortr  r!  Z_thnn_fused_lstm_cellr&  r0  r3  r5  r6  r7  Zargminr8  r9  Ztopkr<  r�   rB  rA  rC  rD  Zpixel_shufflerF  rG  Z	bucketizeZ
Tensor_outrI  ZhistcrN  Z_upsample_bilinear2d_aaZ_upsample_bicubic2d_aarO  rP  rR  r¶  rT  ZsearchsortedrW  rX  Z_embedding_bag_dense_backwardrZ  Z*_embedding_bag_per_sample_weights_backwardr[  Úisinr_  Z	polygammar`  Zchannel_shufflera  Z_local_scalar_denserb  rh  ri  Zspecial_airy_aiZspecial_bessel_y0Zspecial_bessel_y1Zspecial_modified_bessel_i0Zspecial_modified_bessel_i1Zspecial_modified_bessel_k0Zspecial_modified_bessel_k1Z!special_scaled_modified_bessel_k0Z!special_scaled_modified_bessel_k1Zspecial_chebyshev_polynomial_tZspecial_chebyshev_polynomial_uZspecial_chebyshev_polynomial_vZspecial_chebyshev_polynomial_wZ&special_shifted_chebyshev_polynomial_tZ&special_shifted_chebyshev_polynomial_uZ&special_shifted_chebyshev_polynomial_vZ&special_shifted_chebyshev_polynomial_wZspecial_hermite_polynomial_hZspecial_hermite_polynomial_heZspecial_laguerre_polynomial_lZspecial_legendre_polynomial_pZtorch._refs.nn.functionalZtorch._refs.specialrz  r)   r)   r)   r*   Ú<module>   sö  (þ	÷8ÿ	6

   ÿ   ÿø
	

  úú!üüøø"   úú$
ùø
ùø

÷ö#
	
	










ú
	

	



ý)ÿ  ÿüÿ   ÿ ýý


ÿ2

üûÿ   þ&
&
 ÿ  þúù7
 üû(þ
ý 
  üü& þ  
þ÷ö;ùø  û
ú/

 ûúZ
&5üÿ ?üÿ'  



 óøøe
÷,
  ÿ
  ÿ  ÿ"  ÿ   ú    ùM,H
    ùT
N


.


*
$cðð$#
     ÷h	


úÿ


ùÿ

üÿ


!
Têê]?í6G+      ÿ!T
7/


   úg
e(

	    ú,1	




üÿ



üÿÿÿññÿÿ	   ùù"ÿÿ öö*ÿÿ óó7ÿÿ  ïïÿÿ  îî7     øø6ÿÿþÿ
 ûûÿ


üÿ"  ÿ
7'
"ÿ  ÿ
   ÿ

ö
ÿ

	


C