wiki-cased / tools /segment_th.py
Quagmire1's picture
Upload folder using huggingface_hub
41f6dd8 verified
# Copyright (c) 2019-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
#
import sys
from pythainlp.tokenize import word_tokenize
for line in sys.stdin.readlines():
line = line.rstrip('\n')
print(' '.join(word_tokenize(line)))