fix quantization errors of ConvTranspose with per_channel=True

mo-ja · Feb 28, 2024 · 9f42bf8 · 9f42bf8
1 parent 8f5c79c
commit 9f42bf8
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/onnxruntime/python/tools/quantization/operators/conv.py b/onnxruntime/python/tools/quantization/operators/conv.py
@@ -247,7 +247,8 @@ def quantize(self):
             self.quantizer.quantize_activation_tensor(node.output[0])
 
         if self.quantizer.is_per_channel():
-            self.quantizer.quantize_weight_tensor_per_channel(node.input[1], 0)
+            axis = 0 if node.op_type == "Conv" else 1
+            self.quantizer.quantize_weight_tensor_per_channel(node.input[1], axis)
         else:
             self.quantizer.quantize_weight_tensor(node.input[1])