U j@sUddlmZddlmZddlmZddlmZmZm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZmZmZddlmZmZmZm Z m!Z!m"Z"m#Z#eeBeBeBeBZ$de%d<Gd d d Z&ed d d d dddZ'dde(dDZ)de%d<GdddZ*Gddde*Z+Gddde*Z,Gddde*Z-Gddde*Z.Gd d!d!e*Z/Gd"d#d#e*Z0Gd$d%d%e*Z1Gd&d'd'e*Z2Gd(d)d)e*Z3Gd*d+d+e*Z4ed d d,d,d-d.d/d0Z5d7d d3d-d3d4d5d6Z6d S)8) annotations) lru_cache) getLogger)COMMON_CJK_CHARACTERSCOMMON_SAFE_ASCII_CHARACTERSTRACECompatibleFamillyRange _ACCENTUATED_ARABIC_ARABIC_ISOLATED_FORM&_BASIC_LATIN_COMPATIBLE_RANGE_FAMILIES_CJK_COMPATIBLE_RANGE_FAMILIES#_COMPATIBLE_WITH_ANY_RANGE_FAMILIES_HANGUL_HALFWIDTH_KATAKANA _HIRAGANA _KATAKANA_LATIN _LIGATURE_RANGE_FAMILIES_SENTENCE_OPEN_PUNCTUATION _SUPERSCRIPT_THAI)_character_flags is_emoticonis_punctuation is_separator is_symbol remove_accent unicode_rangeint _GLYPH_MASKc@seZdZUdZdZded<ded<ded<ded<ded <ded <ded <ded <ded <ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded<ded <ded!<ded"<dd#d$d%d&Zd'S)(CharInfoz>Pre-computed character properties shared across all detectors.) character printablealphaupperlowerspacedigitis_ascii case_variableflags accentuatedlatinis_cjk is_katakanais_halfwidth_katakana is_arabic is_ligatureis_superscriptis_sentence_open_punctuationis_glyphpunctsymrangesepemoticonsafe common_cjk unaccentedstrr%boolr&r'r(r)r*r+r,r-r"r.r/r0r1r2r3r4r5r6r7r8r9r: str | Noner;r<r=r>r?r@Noner%returncCs||_t|}|dkrfd|_d|_||_d|_d|_|tk|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|krdkrnnFd|_d|_d|_d|_d|_d|_d|_t|_d|_d|_d|_qd|krdkr.nnFd|_d|_d|_d|_d|_d|_d|_t|_d|_d|_d|_qd|krFd krnnDd|_d|_d|_d|_d|_d|_d|_d |_d|_d|_d|_n|d ksd |krd krnnHd|_d|_d|_d|_d|_|d k|_d|_d |_d|_d|_d|_nf||_d|_d|_d|_d|_d|_d|_d |_d|_|jrFt|nd|_|jr\t|nd|_nhd|_d|_||_| |_|!|_|"|_|#|_|$|_|j|jk|_t%|}|jrd|_n t&||_||_t'|t(@|_t'|t@|_t'|t)@|_ t'|t*@|_ t'|t+@|_ t'|t,@|_ t'|t-@|_ t'|t.@|_t'|t/@|_t'|t0@|_|jr|jrt1||_n||_|j o|t2k|_|jrt||_t||_n d|_d|_t3||_4t5||_6dS)zkCompute all properties for *character* (built once per codepoint, every branch assigns every slot).TFAZaz09r N)7r%ordr,r/r@r=r?rr>r1r2r3r4r5r6r7r8r'r(r)r*r+r&r-rr.r0r9r: isprintablerrisalphaisupperislowerisspaceisdigitrrrBr rrrr rrrr#r rr!r;rr<)selfr%or.rZ9/tmp/pip-unpacked-wheel-bwd693va/charset_normalizer/md.py__init__ks    &             zCharInfo.__init__N)__name__ __module__ __qualname____doc__ __slots____annotations__r\rZrZrZr[r$,s> r$N)maxsizerArEcCst|S)zBBuild (once per codepoint) and cache the CharInfo for *character*.)r$)r%rZrZr[ _char_infosrdcCsg|]}tt|qSrZ)r$chr).0Z _codepointrZrZr[ srgrGzlist[CharInfo]_ASCII_CHAR_INFOc@sFeZdZdZdZddddddZdd d d Zed d d dZdS)MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. rZrAr$rDr%inforFcCstdS)z The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. NNotImplementedErrorrXr%rkrZrZr[ feed_info szMessDetectorPlugin.feed_inforFcCstdS)zB Permit to reset the plugin to the initial state. NrlrXrZrZr[resetszMessDetectorPlugin.resetfloatcCstdS)z Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. NrlrqrZrZr[ratioszMessDetectorPlugin.ratioN) r]r^r_r`rarorrpropertyrtrZrZrZr[ris ric@sPeZdZdZddddZddddd d Zddd d Zed dddZdS) TooManySymbolOrPunctuationPlugin_punctuation_count _symbol_count_character_count_last_printable_charrDrpcCsd|_d|_d|_d|_dSNrrwrqrZrZr[r\*sz)TooManySymbolOrPunctuationPlugin.__init__rAr$rjcCs^|jd7_||jkrT|jsT|jr4|jd7_n |jsT|jrT|jsT|jd7_||_dS)1Optimized feed using pre-computed character info.rN) rzr{r>r9rxr+r:r=ryrnrZrZr[ro1sz*TooManySymbolOrPunctuationPlugin.feed_infocCsd|_d|_d|_dSr|)rxrzryrqrZrZr[rr=sz&TooManySymbolOrPunctuationPlugin.resetrscCs0|jdkrdS|j|j|j}|dkr,|SdS)Nr333333?)rzrxry)rXZratio_of_punctuationrZrZr[rtBs   z&TooManySymbolOrPunctuationPlugin.ratioN r]r^r_rar\rorrrurtrZrZrZr[rv"s  rvc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)TooManyAccentuatedPluginrz_accentuated_countrDrpcCsd|_d|_dSr|rrqrZrZr[r\Qsz!TooManyAccentuatedPlugin.__init__rAr$rjcCs&|jd7_|jr"|jd7_dSr}rN)rzr/rrnrZrZr[roUsz"TooManyAccentuatedPlugin.feed_infocCsd|_d|_dSr|rrqrZrZr[rr\szTooManyAccentuatedPlugin.resetrscCs*|jdkrdS|j|j}|dkr&|SdS)Nrgffffff?r)rXZratio_of_accentuationrZrZr[rt`s  zTooManyAccentuatedPlugin.ratioNrrZrZrZr[rNs rc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)UnprintablePlugin_unprintable_countrz _has_escaperDrpcCsd|_d|_d|_dSNrFrrqrZrZr[r\lszUnprintablePlugin.__init__rAr$rjcCsJ|dkrd|_|js8|js8|dkr8|dkr8|jd7_|jd7_dS)r}TurN)rr&r*rrzrnrZrZr[roqszUnprintablePlugin.feed_infocCsd|_d|_dSr)rrrqrZrZr[rrszUnprintablePlugin.resetrscCs(|jdkrdS|jrdS|jd|jS)Nrr?r)rzrrrqrZrZr[rts  zUnprintablePlugin.ratioNrrZrZrZr[ris rc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)SuspiciousDuplicateAccentPlugin_successive_countrz_last_latin_character_last_was_accentuatedrDrpcCsd|_d|_d|_d|_dSrrrqrZrZr[r\sz(SuspiciousDuplicateAccentPlugin.__init__rAr$rjcCsn|jd7_|jdk r\|jr\|jr\|jr@|jjr@|jd7_|j|jjkr\|jd7_||_|j|_dSr)rzrr/rr(rr@rnrZrZr[rosz)SuspiciousDuplicateAccentPlugin.feed_infocCsd|_d|_d|_d|_dSrrrqrZrZr[rrsz%SuspiciousDuplicateAccentPlugin.resetrscCs|jdkrdS|jd|jS)Nrrr~)rzrrqrZrZr[rts z%SuspiciousDuplicateAccentPlugin.ratioNrrZrZrZr[rs rc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)SuspiciousRange"_suspicious_successive_range_countrz_last_printable_seen_last_printable_rangerDrpcCsd|_d|_d|_d|_dSr|rrqrZrZr[r\szSuspiciousRange.__init__rAr$rjcCs|jd7_|js |js |jr0d|_d|_dS|jdkrL||_|j|_dS|j}|j}||ksh|dkrt||r|jd7_||_||_dSr) rzr*r9r>rrr; is_suspiciously_successive_ranger)rXr%rkunicode_range_aunicode_range_brZrZr[ros   zSuspiciousRange.feed_infocCsd|_d|_d|_d|_dSr|)rzrrrrqrZrZr[rrszSuspiciousRange.resetrscCs"|jdkrdS|jd|j}|S)NrPrr~)rzr)rXZratio_of_suspicious_range_usagerZrZr[rts  zSuspiciousRange.ratioNrrZrZrZr[rs rc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)SuperWeirdWordPlugin _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchrz_bad_character_count_buffer_length_buffer_last_char_buffer_last_char_accentuated_buffer_accent_count_buffer_glyph_count_buffer_upper_count_buffer_first_lower_buffer_has_non_ascii_buffer_last_char_ligature_buffer_has_internal_ligature_is_current_word_invalid_invalid_word_countrDrpcCsvd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|_d|_dSrrrqrZrZr[r\ s&zSuperWeirdWordPlugin.__init__rAr$rjcCs|jr|jrd|_|j|_|jdkr,|j|_|jd7_||_|jrT|j d7_ |j s`d|_ |j |_ |j r||jd7_|jr|jd7_n|js|jr|j rd|_dS|jsdS|js|jr|jrd|_d|_|js|js|jr|jd7_|j}|j|7_|dkr|j|dkr.d|_n|j rd|jrd|j |krd|jd7_d|_nV|jdkrd|_|jd7_n4|j r|jr|j |dkr|jd7_d|_|dkr|jr|j dko|j |dk}|s|jd7_d|_|jr*|jd7_|j|7_d |_|jrF|j d7_ d |_d |_d|_d|_d |_ d|_d|_d|_ d |_d |_ d |_d |_n:|d kr|j!s|j"rd|_|jd7_||_d |_ dS) r}TrrN?rF>|=~_-<>)#r'rrr5rr)rrr(rr,rr/rrr8rrr0r7r6rrr*r9r<rrzrTrrrrr+r:)rXr%rkZ buffer_lengthZprobable_camel_casedrZrZr[ro"s      zSuperWeirdWordPlugin.feed_infocCsvd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_d|_d|_d|_d|_dSr)rrrrrrrrzrrrrrrrrrrrrqrZrZr[rrs&zSuperWeirdWordPlugin.resetrscCs.|jr dS|jdkr"|jdkr"dS|j|jS)Nr rr)rrrrrzrqrZrZr[rts zSuperWeirdWordPlugin.ratioNrrZrZrZr[rs drc@sTeZdZdZdZddddZdddd d d Zddd d ZeddddZ dS)CjkUncommonPluginz< Detect messy CJK text that probably means nothing. rz_uncommon_countrDrpcCsd|_d|_dSr|rrqrZrZr[r\szCjkUncommonPlugin.__init__rAr$rjcCs&|jd7_|js"|jd7_dSr)rzr?rrnrZrZr[roszCjkUncommonPlugin.feed_infocCsd|_d|_dSr|rrqrZrZr[rrszCjkUncommonPlugin.resetrscCs.|jdkrdS|j|j}|dkr*|dSdS)Nrrrr)rXZuncommon_form_usagerZrZr[rts  zCjkUncommonPlugin.ratioN r]r^r_r`rar\rorrrurtrZrZrZr[rsrc@sTeZdZdZdZddddZdddd d d Zddd d ZeddddZ dS)SuspiciousKatakanaPluginzDDetect implausible halfwidth Katakana and uncommon CJK combinations._katakana_count_halfwidth_katakana_count _cjk_count_uncommon_cjk_countrDrpcCsd|_d|_d|_d|_dSr|rrqrZrZr[r\sz!SuspiciousKatakanaPlugin.__init__rAr$rjcCsR|jr,|jd7_|jr(|jd7_dS|jd7_|jsN|jd7_dSr)r2rr3rrr?rrnrZrZr[rosz"SuspiciousKatakanaPlugin.feed_infocCsd|_d|_d|_d|_dSr|rrqrZrZr[rrszSuspiciousKatakanaPlugin.resetrscCs:|jdkr6|j|jkr6d|jkr.|jkr6nndSdS)Nrrr)rrrrrqrZrZr[rts  zSuspiciousKatakanaPlugin.ratioNrrZrZrZr[rs rc@sPeZdZdZddddZddddd d Zddd d Zed dddZdS)ArchaicUpperLowerPlugin_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalrz_last_alpha_seen_upper_last_alpha_seen_lower_current_ascii_onlyrDrpcCs4d|_d|_d|_d|_d|_d|_d|_d|_dS)NFrTrrqrZrZr[r\sz ArchaicUpperLowerPlugin.__init__rAr$rjcCs|jo |j}| }|rp|jdkrp|jdkrF|jsF|jsF|j|j7_d|_d|_d|_|jd7_d|_dS|jr|j sd|_|jdkr|j r|j s|j r|j r|jr|jd7_d|_qd|_nd|_|jd7_|jd7_|j |_ |j |_ dS)r}r@FrTNr~)r'r-rr+rrrrrzr,r(rr)r)rXr%rkZ is_concernedZ chunk_seprZrZr[rosF    z!ArchaicUpperLowerPlugin.feed_infocCs4d|_d|_d|_d|_d|_d|_d|_d|_dS)NrFT)rzrrrrrrrrqrZrZr[rr;szArchaicUpperLowerPlugin.resetrscCs|jdkrdS|j|jS)Nrr)rzrrqrZrZr[rtEs zArchaicUpperLowerPlugin.ratioNrrZrZrZr[rs  + rc@sPeZdZdZddddZddddZdd dd d d Zed dddZdS)ArabicIsolatedFormPluginrz_isolated_form_countrDrpcCsd|_d|_dSr|rrqrZrZr[r\Psz!ArabicIsolatedFormPlugin.__init__cCsd|_d|_dSr|rrqrZrZr[rrTszArabicIsolatedFormPlugin.resetrAr$rjcCs*|jd7_|jt@r&|jd7_dSr)rzr.r rrnrZrZr[roXs z"ArabicIsolatedFormPlugin.feed_inforscCs|jdkrdS|j|j}|S)Nrrr)rXZisolated_form_usagerZrZr[rt_s  zArabicIsolatedFormPlugin.ratioN) r]r^r_rar\rrrorurtrZrZrZr[rMs rrCrB)rrrFcCsz|dks|dkrdSt|}t|}||kr0dS|tks@|tkrDdSt||tkrVdS|dkrf|tkS|dkrv|tkSdS)z_ Determine if two Unicode ranges seen next to each other can be considered suspicious. NTFz Basic Latin)rrr rr )rrZ familly_aZ familly_brZrZr[ris$r皙?Frs)decoded_sequencemaximum_thresholddebugrFc$ Cst|}|dkrd}n|dkr$d}nd}|}t}t}t}t} t} t} t} t } t }t }t }t }|j}| j}| j}| j}| j}| j}|j}|j}|j}|j}td||D]:}||||D]}t|}|dkr||}n||}|||||||r$|jr|||q||||jrJ|||||||jr||||jrl||||jr||||||n|jr||||jr|||q|j| j| j| j| j| j|j|j|j|j} | |krqbq|d}!|d|!|s|d|!|d|!|j| j| j| j| j| j|j|j|j|j} |rtd }"|"td |d | d ||d kr|"td|dd |"td|dd|| | | | | ||||f D] }#|"t|#jd|#jqt| dS)zw Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. irNirrGrr Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=zStarting with: Nz Ending with: iz: r)lenisasciirhrdrvrrrrrrrrrror;rQr&r'r0r1r2r4rtrlogr __class__round)$rrrZseq_lenstepZ is_pure_asciiZ ascii_infoZ char_infoZd_spZd_taZd_upZd_sdaZd_srZd_swZd_cuZd_skZd_auZd_aiZ d_sp_feedZ d_ta_feedZ d_up_feedZ d_sda_feedZ d_sr_feedZ d_sw_feedZ d_cu_feedZ d_sk_feedZ d_au_feedZ d_ai_feedZ block_startr% codepointrkZmean_mess_ratioZnl_infologgerdtrZrZr[ mess_ratios                      r)rF)7 __future__r functoolsrloggingrZconstantrrrr r r r r rrrrrrrrrrrrrutilsrrrrrr r!r#rbr$rdr;rhrirvrrrrrrrrrrrrZrZrZr[s<   \$ M,%,94".W$