transform_landmarks, transform_tensor_bilinear, landmarks_to_transform_matrix

Author: PINTO0309

README.md

@@ -142,6 +142,9 @@ Generate saved_model, tfjs, tf-trt, EdgeTPU, CoreML, quantized tflite, ONNX, Ope
 |125|Densify|(const)||
 |126|SPACE_TO_BATCH_ND|tf.space_to_batch_nd||
 |127|BATCH_TO_SPACE_ND|tf.compat.v1.batch_to_space_nd||
+|128|TransformLandmarks|tf.reshape, tf.linalg.matmul, tf.math.add|CUSTOM, MediaPipe|
+|129|TransformTensorBilinear|tf.reshape, tf.linalg.matmul, tf.math.add, tf.tile, tf.math.floor, tf.math.subtract, tf.math.multiply, tf.math.reduce_prod, tf.cast, tf.math.maximum, tf.math.maximum, tf.concat, tf.gather_nd|CUSTOM, MediaPipe|
+|130|Landmarks2TransformMatrix|tf.constant, tf.math.subtract, tf.math.norm, tf.math.divide, tf.linalg.matmul, tf.concat, tf.transpose, tf.gather, tf.math.reduce_min, tf.math.reduce_max, tf.math.multiply, tf.zeros, tf.math.add, tf.tile|CUSTOM, MediaPipe|
 
 ## 2. Environment
 - Python3.8+
@@ -549,7 +552,9 @@ $ tflite2tensorflow \
 $ view_npy --npy_file_path calibration_data_img_sample.npy
 ```
 Press the **`Q`** button to display the next image. **`calibration_data_img_sample.npy`** contains 20 images extracted from the MS-COCO data set.
-![ezgif com-gif-maker](https://user-images.githubusercontent.com/33194443/109318923-aba15480-7891-11eb-84aa-034f77125f34.gif)
+
+![image](https://user-images.githubusercontent.com/33194443/160409583-66c45d47-636b-442c-94d6-51ad4170cc9b.png)
+
 ## 5. Sample image
 This is the result of converting MediaPipe's Meet Segmentation model (segm_full_v679.tflite / Float16 / Google Meet) to **`saved_model`** and then reconverting it to Float32 tflite. Replace the GPU-optimized **`Convolution2DTransposeBias`** layer with the standard **`TransposeConv`** and **`BiasAdd`** layers in a fully automatic manner. The weights and biases of the Float16 **`Dequantize`** layer are automatically back-quantized to Float32 precision. The generated **`saved_model`** in Float32 precision can be easily converted to **`Float16`**, **`INT8`**, **`EdgeTPU`**, **`TFJS`**, **`TF-TRT`**, **`CoreML`**, **`ONNX`**, **`OpenVINO`**, **`Myriad Inference Engine blob`**.
 

setup.py

@@ -11,7 +11,7 @@
     setup(
         name="tflite2tensorflow",
         scripts=scripts,
-        version="1.20.5",
+        version="1.20.6",
         description="Generate saved_model, tfjs, tf-trt, EdgeTPU, CoreML, quantized tflite, ONNX, OpenVINO, Myriad Inference Engine blob and .pb from .tflite.",
         long_description=long_description,
         long_description_content_type="text/markdown",

tflite2tensorflow/__init__.py

@@ -3,19 +3,19 @@
 ###############################################################################
 #
 # MIT License
-# 
+#
 # Copyright (c) 2022 Akiya Research Institute, Inc.
-# 
+#
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
 # in the Software without restriction, including without limitation the rights
 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the Software is
 # furnished to do so, subject to the following conditions:
-# 
+#
 # The above copyright notice and this permission notice shall be included in all
 # copies or substantial portions of the Software.
-# 
+#
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
@@ -87,7 +87,7 @@ def TransformTensorBilinear(operator, custom_options, tensors, interpreter, feat
     weight_ceil_ = tf.reduce_prod(weight_ceil_, axis=3, keepdims=True) #[b,h,w,1]
     weight_floor = tf.reduce_prod(weight_floor, axis=3, keepdims=True) #[b,h,w,1]
 
-    # Find nearest 4 points. 
+    # Find nearest 4 points.
     # Make sure they are in the input image
     in_coord_floor = tf.cast(in_coord_floor, dtype=tf.int32) #[b,h,w,XY]
     in_coord_floor = tf.maximum(in_coord_floor, tf.zeros(2, dtype=tf.int32)) #[b,h,w,XY]
@@ -97,7 +97,7 @@ def TransformTensorBilinear(operator, custom_options, tensors, interpreter, feat
     # in_coord_ceil_ = tf.maximum(in_coord_ceil_, tf.zeros(2, dtype=tf.int32)) #[b,h,w,XY]
     in_coord_ceil_ = tf.minimum(in_coord_ceil_, [input_w, input_h]) #[b,h,w,XY]
 
-    in_coord_ceilX = tf.concat([in_coord_floor[:,:,:,1:2], in_coord_ceil_[:,:,:,0:1]], axis=3) #[b,h,w,YX] YX for BHWC 
+    in_coord_ceilX = tf.concat([in_coord_floor[:,:,:,1:2], in_coord_ceil_[:,:,:,0:1]], axis=3) #[b,h,w,YX] YX for BHWC
     in_coord_ceilY = tf.concat([in_coord_ceil_[:,:,:,1:2], in_coord_floor[:,:,:,0:1]], axis=3) #[b,h,w,YX]
     in_coord_floor = tf.concat([in_coord_floor[:,:,:,1:2], in_coord_floor[:,:,:,0:1]], axis=3) #[b,h,w,YX]
     in_coord_ceil_ = tf.concat([in_coord_ceil_[:,:,:,1:2], in_coord_ceil_[:,:,:,0:1]], axis=3) #[b,h,w,YX]
@@ -122,7 +122,7 @@ def TransformTensorBilinear(operator, custom_options, tensors, interpreter, feat
 
 # Left indexとRight indexで指定されたLandmarkを結ぶ線が水平になり、Subset indicesで指定されたLandmrakをちょうど含むような範囲をcropするように、元の画像をAffine変換する行列
 # の逆行列を求める。なぜ、逆行列かといういうと、後の計算で使うのが逆行列だから。
-# Calc inverse of the matrix which represetns the affine transform which crops the area which covers all the landmarks specified by "subset indices" and rotates so that the landmarks specified by "Left index" and "Right index" are horizontally aligned. 
+# Calc inverse of the matrix which represetns the affine transform which crops the area which covers all the landmarks specified by "subset indices" and rotates so that the landmarks specified by "Left index" and "Right index" are horizontally aligned.
 def Landmarks2TransformMatrix(operator, custom_options, tensors, interpreter, landmarks3d=None):
     if landmarks3d is None:
         landmarks3d = tensors[operator['inputs'][0]] #float32 [b,468,3] landmarks
@@ -189,7 +189,7 @@ def Landmarks2TransformMatrix(operator, custom_options, tensors, interpreter, la
     #
     # t = center - shift
     #
-    # shift = -0.5 * output_w * sx * u 
+    # shift = -0.5 * output_w * sx * u
     #       + -0.5 * output_h * sy * v
     sxu = tf.multiply(u, scale[:,:,0:1]) #[b,1,2]
     syv = tf.multiply(v, scale[:,:,1:2]) #[b,1,2]

tflite2tensorflow/mediapipeCustomOp.py

@@ -21,7 +21,7 @@
 import struct
 import itertools
 import pandas as pd
-from mediapipeCustomOp import Landmarks2TransformMatrix, TransformTensorBilinear, TransformLandmarks
+from tflite2tensorflow.mediapipeCustomOp import Landmarks2TransformMatrix, TransformTensorBilinear, TransformLandmarks
 
 class Color:
     BLACK          = '\033[30m'
@@ -5085,18 +5085,18 @@ def complexabs_(x, tout):
                 # MediaPipe v0.8.9
                 elif custom_op_type == 'Landmarks2TransformMatrix':
                     options = op['custom_options']
-                    custom_options = read_flexbuffer(np.array(options, dtype=np.uint8).tobytes())                    
+                    custom_options = read_flexbuffer(np.array(options, dtype=np.uint8).tobytes())
                     output_detail = interpreter._get_tensor_details(op['outputs'][0])
                     tensors[output_detail['index']] = Landmarks2TransformMatrix(op, custom_options, tensors, interpreter)
 
                 elif custom_op_type == 'TransformTensorBilinear':
                     options = op['custom_options']
-                    custom_options = read_flexbuffer(np.array(options, dtype=np.uint8).tobytes())                    
+                    custom_options = read_flexbuffer(np.array(options, dtype=np.uint8).tobytes())
                     output_detail = interpreter._get_tensor_details(op['outputs'][0])
                     tensors[output_detail['index']] = TransformTensorBilinear(op, custom_options, tensors, interpreter)
 
                 elif custom_op_type == 'TransformLandmarks':
-                    custom_options = None                  
+                    custom_options = None
                     output_detail = interpreter._get_tensor_details(op['outputs'][0])
                     tensors[output_detail['index']] = TransformLandmarks(op, custom_options, tensors, interpreter)
 

tflite2tensorflow/tflite2tensorflow.py

@@ -1,3 +1,5 @@
+#! /usr/bin/env python
+
 import numpy as np
 from matplotlib import pyplot as plt
 import argparse