| import argparse |
| import sys |
| from json import dumps |
| from os.path import abspath, basename, dirname, join, realpath |
| from platform import python_version |
| from typing import List, Optional |
| from unicodedata import unidata_version |
|
|
| import charset_normalizer.md as md_module |
| from charset_normalizer import from_fp |
| from charset_normalizer.models import CliDetectionResult |
| from charset_normalizer.version import __version__ |
|
|
|
|
| def query_yes_no(question: str, default: str = "yes") -> bool: |
| """Ask a yes/no question via input() and return their answer. |
| |
| "question" is a string that is presented to the user. |
| "default" is the presumed answer if the user just hits <Enter>. |
| It must be "yes" (the default), "no" or None (meaning |
| an answer is required of the user). |
| |
| The "answer" return value is True for "yes" or False for "no". |
| |
| Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input |
| """ |
| valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} |
| if default is None: |
| prompt = " [y/n] " |
| elif default == "yes": |
| prompt = " [Y/n] " |
| elif default == "no": |
| prompt = " [y/N] " |
| else: |
| raise ValueError("invalid default answer: '%s'" % default) |
|
|
| while True: |
| sys.stdout.write(question + prompt) |
| choice = input().lower() |
| if default is not None and choice == "": |
| return valid[default] |
| elif choice in valid: |
| return valid[choice] |
| else: |
| sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") |
|
|
|
|
| def cli_detect(argv: Optional[List[str]] = None) -> int: |
| """ |
| CLI assistant using ARGV and ArgumentParser |
| :param argv: |
| :return: 0 if everything is fine, anything else equal trouble |
| """ |
| parser = argparse.ArgumentParser( |
| description="The Real First Universal Charset Detector. " |
| "Discover originating encoding used on text file. " |
| "Normalize text to unicode." |
| ) |
|
|
| parser.add_argument( |
| "files", type=argparse.FileType("rb"), nargs="+", help="File(s) to be analysed" |
| ) |
| parser.add_argument( |
| "-v", |
| "--verbose", |
| action="store_true", |
| default=False, |
| dest="verbose", |
| help="Display complementary information about file if any. " |
| "Stdout will contain logs about the detection process.", |
| ) |
| parser.add_argument( |
| "-a", |
| "--with-alternative", |
| action="store_true", |
| default=False, |
| dest="alternatives", |
| help="Output complementary possibilities if any. Top-level JSON WILL be a list.", |
| ) |
| parser.add_argument( |
| "-n", |
| "--normalize", |
| action="store_true", |
| default=False, |
| dest="normalize", |
| help="Permit to normalize input file. If not set, program does not write anything.", |
| ) |
| parser.add_argument( |
| "-m", |
| "--minimal", |
| action="store_true", |
| default=False, |
| dest="minimal", |
| help="Only output the charset detected to STDOUT. Disabling JSON output.", |
| ) |
| parser.add_argument( |
| "-r", |
| "--replace", |
| action="store_true", |
| default=False, |
| dest="replace", |
| help="Replace file when trying to normalize it instead of creating a new one.", |
| ) |
| parser.add_argument( |
| "-f", |
| "--force", |
| action="store_true", |
| default=False, |
| dest="force", |
| help="Replace file without asking if you are sure, use this flag with caution.", |
| ) |
| parser.add_argument( |
| "-t", |
| "--threshold", |
| action="store", |
| default=0.2, |
| type=float, |
| dest="threshold", |
| help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.", |
| ) |
| parser.add_argument( |
| "--version", |
| action="version", |
| version="Charset-Normalizer {} - Python {} - Unicode {} - SpeedUp {}".format( |
| __version__, |
| python_version(), |
| unidata_version, |
| "OFF" if md_module.__file__.lower().endswith(".py") else "ON", |
| ), |
| help="Show version information and exit.", |
| ) |
|
|
| args = parser.parse_args(argv) |
|
|
| if args.replace is True and args.normalize is False: |
| print("Use --replace in addition of --normalize only.", file=sys.stderr) |
| return 1 |
|
|
| if args.force is True and args.replace is False: |
| print("Use --force in addition of --replace only.", file=sys.stderr) |
| return 1 |
|
|
| if args.threshold < 0.0 or args.threshold > 1.0: |
| print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr) |
| return 1 |
|
|
| x_ = [] |
|
|
| for my_file in args.files: |
| matches = from_fp(my_file, threshold=args.threshold, explain=args.verbose) |
|
|
| best_guess = matches.best() |
|
|
| if best_guess is None: |
| print( |
| 'Unable to identify originating encoding for "{}". {}'.format( |
| my_file.name, |
| "Maybe try increasing maximum amount of chaos." |
| if args.threshold < 1.0 |
| else "", |
| ), |
| file=sys.stderr, |
| ) |
| x_.append( |
| CliDetectionResult( |
| abspath(my_file.name), |
| None, |
| [], |
| [], |
| "Unknown", |
| [], |
| False, |
| 1.0, |
| 0.0, |
| None, |
| True, |
| ) |
| ) |
| else: |
| x_.append( |
| CliDetectionResult( |
| abspath(my_file.name), |
| best_guess.encoding, |
| best_guess.encoding_aliases, |
| [ |
| cp |
| for cp in best_guess.could_be_from_charset |
| if cp != best_guess.encoding |
| ], |
| best_guess.language, |
| best_guess.alphabets, |
| best_guess.bom, |
| best_guess.percent_chaos, |
| best_guess.percent_coherence, |
| None, |
| True, |
| ) |
| ) |
|
|
| if len(matches) > 1 and args.alternatives: |
| for el in matches: |
| if el != best_guess: |
| x_.append( |
| CliDetectionResult( |
| abspath(my_file.name), |
| el.encoding, |
| el.encoding_aliases, |
| [ |
| cp |
| for cp in el.could_be_from_charset |
| if cp != el.encoding |
| ], |
| el.language, |
| el.alphabets, |
| el.bom, |
| el.percent_chaos, |
| el.percent_coherence, |
| None, |
| False, |
| ) |
| ) |
|
|
| if args.normalize is True: |
| if best_guess.encoding.startswith("utf") is True: |
| print( |
| '"{}" file does not need to be normalized, as it already came from unicode.'.format( |
| my_file.name |
| ), |
| file=sys.stderr, |
| ) |
| if my_file.closed is False: |
| my_file.close() |
| continue |
|
|
| dir_path = dirname(realpath(my_file.name)) |
| file_name = basename(realpath(my_file.name)) |
|
|
| o_: List[str] = file_name.split(".") |
|
|
| if args.replace is False: |
| o_.insert(-1, best_guess.encoding) |
| if my_file.closed is False: |
| my_file.close() |
| elif ( |
| args.force is False |
| and query_yes_no( |
| 'Are you sure to normalize "{}" by replacing it ?'.format( |
| my_file.name |
| ), |
| "no", |
| ) |
| is False |
| ): |
| if my_file.closed is False: |
| my_file.close() |
| continue |
|
|
| try: |
| x_[0].unicode_path = join(dir_path, ".".join(o_)) |
|
|
| with open(x_[0].unicode_path, "w", encoding="utf-8") as fp: |
| fp.write(str(best_guess)) |
| except IOError as e: |
| print(str(e), file=sys.stderr) |
| if my_file.closed is False: |
| my_file.close() |
| return 2 |
|
|
| if my_file.closed is False: |
| my_file.close() |
|
|
| if args.minimal is False: |
| print( |
| dumps( |
| [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__, |
| ensure_ascii=True, |
| indent=4, |
| ) |
| ) |
| else: |
| for my_file in args.files: |
| print( |
| ", ".join( |
| [ |
| el.encoding or "undefined" |
| for el in x_ |
| if el.path == abspath(my_file.name) |
| ] |
| ) |
| ) |
|
|
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| cli_detect() |
|
|
