upgrade sonnx for nlp and cv models

examples/onnx/arcface.py

@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import os
+import numpy as np
+from PIL import Image
+from sklearn import preprocessing
+
+from singa import device
+from singa import tensor
+from singa import autograd
+from singa import sonnx
+import onnx
+from utils import download_model, update_batch_size, check_exist_or_download
+
+import logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+
+def preprocess(img):
+    w, h = img.size
+    img = img.crop((0, (h - w) // 2, w, h - (h - w) // 2))
+    img = img.resize((112, 112))
+    img = np.array(img).astype(np.float32)
+    img = np.rollaxis(img, 2, 0)
+    img = np.expand_dims(img, axis=0)
+    return img
+
+
+def get_image():
+    # download image
+    img1 = Image.open(
+        check_exist_or_download(
+            'https://angus-doc.readthedocs.io/en/latest/_images/aurelien.jpg'))
+    img2 = Image.open(
+        check_exist_or_download(
+            'https://angus-doc.readthedocs.io/en/latest/_images/gwenn.jpg'))
+    return img1, img2
+
+
+class Infer:
+
+    def __init__(self, sg_ir):
+        self.sg_ir = sg_ir
+        for idx, tens in sg_ir.tensor_map.items():
+            # allow the tensors to be updated
+            tens.requires_grad = True
+            tens.stores_grad = True
+            sg_ir.tensor_map[idx] = tens
+
+    def forward(self, x):
+        return sg_ir.run([x])[0]
+
+
+if __name__ == "__main__":
+
+    download_dir = '/tmp'
+    url = 'https://s3.amazonaws.com/onnx-model-zoo/arcface/resnet100/resnet100.tar.gz'
+    model_path = os.path.join(download_dir, 'resnet100', 'resnet100.onnx')
+
+    logging.info("onnx load model...")
+    download_model(url)
+    onnx_model = onnx.load(model_path)
+
+    # set batch size
+    onnx_model = update_batch_size(onnx_model, 2)
+
+    # prepare the model
+    logging.info("prepare model...")
+    dev = device.create_cuda_gpu()
+    sg_ir = sonnx.prepare(onnx_model, device=dev)
+    autograd.training = False
+    model = Infer(sg_ir)
+
+    # verifty the test dataset
+    # from utils import load_dataset
+    # inputs, ref_outputs = load_dataset(
+    #     os.path.join('/tmp', 'resnet100', 'test_data_set_0'))
+    # x_batch = tensor.Tensor(device=dev, data=inputs[0])
+    # outputs = model.forward(x_batch)
+    # for ref_o, o in zip(ref_outputs, outputs):
+    #     np.testing.assert_almost_equal(ref_o, tensor.to_numpy(o), 4)
+
+    # inference demo
+    logging.info("preprocessing...")
+    img1, img2 = get_image()
+    img1 = preprocess(img1)
+    img2 = preprocess(img2)
+
+    x_batch = tensor.Tensor(device=dev,
+                            data=np.concatenate((img1, img2), axis=0))
+    logging.info("model running...")
+    y = model.forward(x_batch)
+
+    logging.info("postprocessing...")
+    embedding = tensor.to_numpy(y)
+    embedding = preprocessing.normalize(embedding)
+    embedding1 = embedding[0]
+    embedding2 = embedding[1]
+
+    # Compute squared distance between embeddings
+    dist = np.sum(np.square(embedding1 - embedding2))
+    # Compute cosine similarity between embedddings
+    sim = np.dot(embedding1, embedding2.T)
+    # logging.info predictions
+    logging.info('Distance = %f' % (dist))
+    logging.info('Similarity = %f' % (sim))

examples/onnx/bert/bert-squad.py

@@ -0,0 +1,169 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under th
+
+import os
+import zipfile
+import numpy as np
+import json
+
+from singa import device
+from singa import tensor
+from singa import sonnx
+from singa import autograd
+import onnx
+import tokenization
+from run_onnx_squad import read_squad_examples, convert_examples_to_features, RawResult, write_predictions
+
+import sys
+sys.path.append(os.path.dirname(__file__) + '/..')
+from utils import download_model, update_batch_size, check_exist_or_download
+
+import logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)-15s %(message)s')
+
+max_answer_length = 30
+max_seq_length = 256
+doc_stride = 128
+max_query_length = 64
+n_best_size = 20
+batch_size = 1
+
+
+def load_vocab():
+    url = 'https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip'
+    download_dir = '/tmp/'
+    filename = os.path.join(download_dir, 'uncased_L-12_H-768_A-12', '.',
+                            'vocab.txt')
+    with zipfile.ZipFile(check_exist_or_download(url), 'r') as z:
+        z.extractall(path=download_dir)
+    return filename
+
+
+class Infer:
+
+    def __init__(self, sg_ir):
+        self.sg_ir = sg_ir
+
+    def forward(self, x):
+        return sg_ir.run(x)
+
+
+def preprocess():
+    vocab_file = load_vocab()
+    tokenizer = tokenization.FullTokenizer(vocab_file=vocab_file,
+                                           do_lower_case=True)
+    predict_file = os.path.join(os.path.dirname(__file__), 'inputs.json')
+    # print content
+    with open(predict_file) as json_file:
+        test_data = json.load(json_file)
+        print("The input is:", json.dumps(test_data, indent=2))
+
+    eval_examples = read_squad_examples(input_file=predict_file)
+
+    # Use convert_examples_to_features method from run_onnx_squad to get parameters from the input
+    input_ids, input_mask, segment_ids, extra_data = convert_examples_to_features(
+        eval_examples, tokenizer, max_seq_length, doc_stride, max_query_length)
+    return input_ids, input_mask, segment_ids, extra_data, eval_examples
+
+
+def postprocess(eval_examples, extra_data, all_results):
+    output_dir = 'predictions'
+    os.makedirs(output_dir, exist_ok=True)
+    output_prediction_file = os.path.join(output_dir, "predictions.json")
+    output_nbest_file = os.path.join(output_dir, "nbest_predictions.json")
+    write_predictions(eval_examples, extra_data, all_results, n_best_size,
+                      max_answer_length, True, output_prediction_file,
+                      output_nbest_file)
+
+    # print results
+    with open(output_prediction_file) as json_file:
+        test_data = json.load(json_file)
+        print("The result is:", json.dumps(test_data, indent=2))
+
+
+if __name__ == "__main__":
+
+    url = 'https://media.githubusercontent.com/media/onnx/models/master/text/machine_comprehension/bert-squad/model/bertsquad-10.tar.gz'
+    download_dir = '/tmp/'
+    model_path = os.path.join(download_dir, 'download_sample_10',
+                              'bertsquad10.onnx')
+
+    logging.info("onnx load model...")
+    download_model(url)
+    onnx_model = onnx.load(model_path)
+
+    # set batch size
+    onnx_model = update_batch_size(onnx_model, batch_size)
+    dev = device.create_cuda_gpu()
+    autograd.training = False
+
+    # inference
+    logging.info("preprocessing...")
+    input_ids, input_mask, segment_ids, extra_data, eval_examples = preprocess()
+
+    sg_ir = None
+    n = len(input_ids)
+    bs = batch_size
+    all_results = []
+
+    tmp_dict = {}
+    for idx in range(0, n):
+        logging.info("starting infer sample {}...".format(idx))
+        item = eval_examples[idx]
+        inputs = [
+            np.array([item.qas_id], dtype=np.int32),
+            segment_ids[idx:idx + bs].astype(np.int32),
+            input_mask[idx:idx + bs].astype(np.int32),
+            input_ids[idx:idx + bs].astype(np.int32),
+        ]
+
+        if sg_ir is None:
+            # prepare the model
+            logging.info("model is none, prepare model...")
+            sg_ir = sonnx.prepare(onnx_model,
+                                  device=dev,
+                                  init_inputs=inputs,
+                                  keep_initializers_as_inputs=False)
+            model = Infer(sg_ir)
+
+        x_batch = []
+        for inp in inputs:
+            tmp_tensor = tensor.from_numpy(inp)
+            tmp_tensor.to_device(dev)
+            x_batch.append(tmp_tensor)
+
+        logging.info("model running for sample {}...".format(idx))
+        outputs = model.forward(x_batch)
+
+        logging.info("hanlde the result of sample {}...".format(idx))
+        result = []
+        for outp in outputs:
+            result.append(tensor.to_numpy(outp))
+
+        in_batch = result[1].shape[0]
+        start_logits = [float(x) for x in result[1][0].flat]
+        end_logits = [float(x) for x in result[0][0].flat]
+        for i in range(0, in_batch):
+            unique_id = len(all_results)
+            all_results.append(
+                RawResult(unique_id=unique_id,
+                          start_logits=start_logits,
+                          end_logits=end_logits))
+    # postprocessing
+    logging.info("postprocessing...")
+    postprocess(eval_examples, extra_data, all_results)

examples/onnx/bert/inputs.json

@@ -0,0 +1,27 @@
+{
+  "version": "1.4",
+  "data": [
+    {
+      "paragraphs": [
+        {
+          "context": "In its early years, the new convention center failed to meet attendance and revenue expectations.[12] By 2002, many Silicon Valley businesses were choosing the much larger Moscone Center in San Francisco over the San Jose Convention Center due to the latter's limited space. A ballot measure to finance an expansion via a hotel tax failed to reach the required two-thirds majority to pass. In June 2005, Team San Jose built the South Hall, a $6.77 million, blue and white tent, adding 80,000 square feet (7,400 m2) of exhibit space",
+          "qas": [
+            {
+              "question": "where is the businesses choosing to go?",
+              "id": "1"
+            },
+            {
+              "question": "how may votes did the ballot measure need?",
+              "id": "2"
+            },
+            {
+              "question": "By what year many Silicon Valley businesses were choosing the Moscone Center?",
+              "id": "3"
+            }
+          ]
+        }
+      ],
+      "title": "Conference Center"
+    }
+  ]
+}