# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# pylint: disable=import-outside-toplevel
# pylint: disable=c-extension-no-member
# pylint: disable=invalid-name
# pylint: disable=too-many-boolean-expressions
"""
BasicTokenizer
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unicodedata
import platform
import numpy as np
import mindspore._c_dataengine as cde
from mindspore.dataset.transforms.transforms import PyTensorOperation
from mindspore.dataset.text.transforms import TextTensorOperation, Implementation
[docs]class BasicTokenizer(TextTensorOperation, PyTensorOperation):
"""
Tokenize the input UTF-8 encoded string by specific rules.
Args:
lower_case (bool, optional): Whether to perform lowercase processing on the text. If True, will fold the
text to lower case and strip accented characters. If False, will only perform normalization on the
text, with mode specified by `normalization_form`. Default: False.
py_transform (bool, optional): Whether use python implementation. Default: False.
Raises:
TypeError: If `lower_case` is not of type bool.
TypeError: If `py_transform` is not of type bool.
RuntimeError: If dtype of input Tensor is not str.
Supported Platforms:
``CPU``
Examples:
>>> from mindnlp.dataset.transforms import BasicTokenizer
>>> tokenizer_op = BasicTokenizer()
>>> text = "Welcom to China!"
>>> tokenized_text = tokenizer_op(text)
"""
# @check_decode
def __init__(self, lower_case=False, py_transform=False):
super().__init__()
if py_transform or platform.system().lower() == 'windows':
self.tokenizer = _BasicTokenizer(lower_case)
self.implementation = Implementation.PY
else:
self.tokenizer = None
self.implementation = Implementation.C
self.lower_case = lower_case
def __call__(self, text_input):
"""
Call method for input conversion for eager mode with C++ implementation.
"""
if isinstance(text_input, str):
text_input = np.array(text_input)
elif not isinstance(text_input, np.ndarray):
raise TypeError(
f"Input should be a text line in 1-D NumPy format, got {type(text_input)}.")
return super().__call__(text_input)
[docs] def execute_py(self, text_input):
"""
Execute method.
"""
return self._execute_py(text_input)
def _execute_py(self, text_input):
"""
Execute method.
"""
tokens = self.tokenizer.tokenize(text_input)
return np.array(tokens)
[docs] def parse(self):
from mindspore.dataset.text.transforms import DE_C_INTER_NORMALIZE_FORM, NormalizeForm
normalization_form = DE_C_INTER_NORMALIZE_FORM.get(NormalizeForm.NFD)
return cde.BasicTokenizerOperation(self.lower_case, False, normalization_form,
False, False)
def _convert_to_unicode(text):
"""Converts `text` to Unicode (if it's not already), assuming utf-8 input."""
if isinstance(text, str):
return text
if isinstance(text, bytes):
return text.decode("utf-8", "ignore")
if isinstance(text, np.ndarray):
if text.dtype.type is np.bytes_:
text = np.char.decode(text, "utf-8")
return str(text)
raise ValueError(f"Unsupported string type: {type(text)}, {text.dtype}")
def _whitespace_tokenize(text):
"""Runs basic whitespace cleaning and splitting on a piece of text."""
text = text.strip()
if not text:
return []
tokens = text.split()
return tokens
class _BasicTokenizer():
"""Runs basic tokenization (punctuation splitting, lower casing, etc.)."""
def __init__(self, do_lower_case=True):
"""Constructs a BasicTokenizer.
Args:
do_lower_case: Whether to lower case the input.
"""
self.do_lower_case = do_lower_case
def tokenize(self, text):
"""Tokenizes a piece of text."""
text = _convert_to_unicode(text)
text = self._clean_text(text)
# This was added on November 1st, 2018 for the multilingual and Chinese
# models. This is also applied to the English models now, but it doesn't
# matter since the English models were not trained on any Chinese data
# and generally don't have any Chinese data in them (there are Chinese
# characters in the vocabulary because Wikipedia does have some Chinese
# words in the English Wikipedia.).
text = self._tokenize_chinese_chars(text)
orig_tokens = _whitespace_tokenize(text)
split_tokens = []
for token in orig_tokens:
if self.do_lower_case:
token = token.lower()
token = self._run_strip_accents(token)
split_tokens.extend(self._run_split_on_punc(token))
output_tokens = _whitespace_tokenize(" ".join(split_tokens))
return output_tokens
def _run_strip_accents(self, text):
"""Strips accents from a piece of text."""
text = unicodedata.normalize("NFD", text)
output = []
for char in text:
cat = unicodedata.category(char)
if cat == "Mn":
continue
output.append(char)
return "".join(output)
def _run_split_on_punc(self, text):
"""Splits punctuation on a piece of text."""
chars = list(text)
i = 0
start_new_word = True
output = []
while i < len(chars):
char = chars[i]
if _is_punctuation(char):
output.append([char])
start_new_word = True
else:
if start_new_word:
output.append([])
start_new_word = False
output[-1].append(char)
i += 1
return ["".join(x) for x in output]
def _tokenize_chinese_chars(self, text):
"""Adds whitespace around any CJK character."""
output = []
for char in text:
cp = ord(char)
if self._is_chinese_char(cp):
output.append(" ")
output.append(char)
output.append(" ")
else:
output.append(char)
return "".join(output)
def _is_chinese_char(self, cp):
"""Checks whether CP is the codepoint of a CJK character."""
# This defines a "chinese character" as anything in the CJK Unicode block:
# https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
#
# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
# despite its name. The modern Korean Hangul alphabet is a different block,
# as is Japanese Hiragana and Katakana. Those alphabets are used to write
# space-separated words, so they are not treated specially and handled
# like the all of the other languages.
if ((0x4E00 <= cp <= 0x9FFF) or
(0x3400 <= cp <= 0x4DBF) or
(0x20000 <= cp <= 0x2A6DF) or
(0x2A700 <= cp <= 0x2B73F) or
(0x2B740 <= cp <= 0x2B81F) or
(0x2B820 <= cp <= 0x2CEAF) or
(0xF900 <= cp <= 0xFAFF) or
(0x2F800 <= cp <= 0x2FA1F)):
return True
return False
def _clean_text(self, text):
"""Performs invalid character removal and whitespace cleanup on text."""
output = []
for char in text:
cp = ord(char)
if cp == 0 or cp == 0xfffd or _is_control(char):
continue
if _is_whitespace(char):
output.append(" ")
else:
output.append(char)
return "".join(output)
def _is_whitespace(char):
"""Checks whether `chars` is a whitespace character."""
# \t, \n, and \r are technically contorl characters but we treat them
# as whitespace since they are generally considered as such.
if char in (" ", "\t", "\n", "\r"):
return True
cat = unicodedata.category(char)
if cat == "Zs":
return True
return False
def _is_control(char):
"""Checks whether `chars` is a control character."""
# These are technically control characters but we count them as whitespace
# characters.
if char in ("\t", "\n", "\r"):
return False
cat = unicodedata.category(char)
if cat in ("Cc", "Cf"):
return True
return False
def _is_punctuation(char):
"""Checks whether `chars` is a punctuation character."""
cp = ord(char)
# We treat all non-letter/number ASCII as punctuation.
# Characters such as "^", "$", and "`" are not in the Unicode
# Punctuation class but we treat them as punctuation anyways, for
# consistency.
if ((33 <= cp <= 47) or
(58 <= cp <= 64) or
(91 <= cp <= 96) or
(123 <= cp <= 126)):
return True
cat = unicodedata.category(char)
if cat.startswith("P"):
return True
return False