관리-도구
편집 파일: _inputstream.cpython-38.pyc
U ��.e( � ! @ s d dl mZmZmZ d dlmZmZ d dlmZm Z d dl Z d dl Z d dl m Z ddlmZmZmZmZ ddlmZ dd lmZ d d lmZ zd d lmZ W n ek r� eZY nX ed d � eD ��Zedd � eD ��Z edd � eD ��Z eeddg�B Z dZ ej �rJe d dk�r&e �!d�dk�s*t"�e �#e dd� e$d� d �Z%n e �#e �Z%e&ddddddd d d d d d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5g �Z'e �#d6�Z(i Z)G d7d8� d8e*�Z+d9d:� Z,G d;d<� d<e*�Z-G d=d>� d>e-�Z.G d?d@� d@e/�Z0G dAdB� dBe*�Z1G dCdD� dDe*�Z2dEdF� Z3dS )G� )�absolute_import�division�unicode_literals)� text_type� binary_type)� http_client�urllibN)� webencodings� )�EOF�spaceCharacters� asciiLetters�asciiUppercase)�_ReparseException)�_utils)�StringIO)�BytesIOc C s g | ]}|� d ��qS ��ascii��encode��.0�item� r �E/usr/lib/python3.8/site-packages/pip/_vendor/html5lib/_inputstream.py� <listcomp> s r c C s g | ]}|� d ��qS r r r r r r r s c C s g | ]}|� d ��qS r r r r r r r s � >� <u� [- - --]����]z"\uD800-\uDFFF"i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� i�� z[ - -/:-@\[-`{-~]c @ sH e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dS )�BufferedStreamz�Buffering for streams that do not have buffering of their own The buffer is implemented as a list of chunks on the assumption that joining many strings will be slow since it is O(n**2) c C s || _ g | _ddg| _d S )Nr r )�stream�buffer�position)�selfr" r r r �__init__@ s zBufferedStream.__init__c C s< d}| j d | jd � D ]}|t|�7 }q|| jd 7 }|S �Nr r )r# r$ �len)r% �pos�chunkr r r �tellE s zBufferedStream.tellc C sT || � � kst�|}d}t| j| �|k rF|t| j| �8 }|d7 }q||g| _d S r' )�_bufferedBytes�AssertionErrorr( r# r$ )r% r) �offset�ir r r �seekL s zBufferedStream.seekc C sT | j s| �|�S | jd t| j �krF| jd t| j d �krF| �|�S | �|�S d S )Nr r r )r# � _readStreamr$ r( �_readFromBuffer)r% �bytesr r r �readU s � zBufferedStream.readc C s t dd� | jD ��S )Nc S s g | ] }t |��qS r )r( r r r r r _ s z1BufferedStream._bufferedBytes.<locals>.<listcomp>)�sumr# �r% r r r r, ^ s z BufferedStream._bufferedBytesc C s< | j �|�}| j�|� | jd d7 < t|�| jd< |S r' )r" r4 r# �appendr$ r( )r% r3 �datar r r r1 a s zBufferedStream._readStreamc C s� |}g }| j d }| j d }|t| j�k r�|dkr�|dks>t�| j| }|t|�| krl|}||| g| _ n"t|�| }|t|�g| _ |d7 }|�|||| � � ||8 }d}q |r�|�| �|�� d�|�S )Nr r � )r$ r( r# r- r7 r1 �join)r% r3 ZremainingBytes�rvZ bufferIndexZ bufferOffsetZ bufferedDataZ bytesToReadr r r r2 h s& z BufferedStream._readFromBufferN) �__name__� __module__� __qualname__�__doc__r& r+ r0 r4 r, r1 r2 r r r r r! 9 s r! c K s� t | tj�s(t | tjj�r.t | jtj�r.d}n&t| d�rJt | �d�t �}n t | t �}|r�dd� |D �}|rvt d| ��t | f|�S t | f|�S d S )NFr4 r c S s g | ]}|� d �r|�qS )Z _encoding)�endswith)r �xr r r r � s z#HTMLInputStream.<locals>.<listcomp>z3Cannot set an encoding with a unicode input, set %r) � isinstancer Z HTTPResponser ZresponseZaddbase�fp�hasattrr4 r � TypeError�HTMLUnicodeInputStream�HTMLBinaryInputStream)�source�kwargsZ isUnicodeZ encodingsr r r �HTMLInputStream� s � � rJ c @ sp e Zd ZdZdZdd� Zdd� Zdd� Zd d � Zd d � Z d d� Z ddd�Z dd� Z dd� Z d dd�Zdd� ZdS ) rF ��Provides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing incorrect byte-sequences and also provides column and line tracking. i ( c C sZ t jsd| _n td�dkr$| j| _n| j| _dg| _td�df| _| � |�| _ | � � dS )� Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) Nu r r �utf-8�certain) r �supports_lone_surrogates�reportCharacterErrorsr( �characterErrorsUCS4�characterErrorsUCS2ZnewLines�lookupEncoding� charEncoding� openStream� dataStream�reset)r% rH r r r r& � s z HTMLUnicodeInputStream.__init__c C s. d| _ d| _d| _g | _d| _d| _d | _d S )N� r )r* � chunkSize� chunkOffset�errors� prevNumLines� prevNumCols�_bufferedCharacterr6 r r r rW � s z HTMLUnicodeInputStream.resetc C s t |d�r|}nt|�}|S �zvProduces a file object from source. source can be either a file object, local filename or a string. r4 )rD r �r% rH r" r r r rU � s z!HTMLUnicodeInputStream.openStreamc C sT | j }|�dd|�}| j| }|�dd|�}|dkr@| j| }n ||d }||fS )N� r r r )r* �countr\ �rfindr] )r% r. r* ZnLinesZ positionLineZ lastLinePosZpositionColumnr r r � _position� s z HTMLUnicodeInputStream._positionc C s | � | j�\}}|d |fS )z:Returns (line, col) of the current position in the stream.r )rd rZ )r% �line�colr r r r$ � s z HTMLUnicodeInputStream.positionc C s6 | j | jkr| �� stS | j }| j| }|d | _ |S )zo Read one character from the stream or queue if available. Return EOF when EOF is reached. r )rZ rY � readChunkr r* )r% rZ �charr r r rh � s zHTMLUnicodeInputStream.charNc C s� |d kr| j }| �| j�\| _| _d| _d| _d| _| j�|�}| j rX| j | }d | _ n|s`dS t |�dkr�t |d �}|dks�d| kr�dkr�n n|d | _ |d d� }| j r�| � |� |� d d �}|� d d �}|| _t |�| _d S ) NrX r Fr r � � � i�� z ra � T)�_defaultChunkSizerd rY r\ r] r* rZ rV r4 r^ r( �ordrP �replace)r% rY r8 Zlastvr r r rg � s0 z HTMLUnicodeInputStream.readChunkc C s( t tt�|���D ]}| j�d� qd S )N�invalid-codepoint)�ranger( �invalid_unicode_re�findallr[ r7 )r% r8 �_r r r rQ % s z*HTMLUnicodeInputStream.characterErrorsUCS4c C s� d}t �|�D ]�}|rqt|�� �}|�� }t�|||d � �rrt�|||d � �}|tkrl| j � d� d}q|dkr�|dkr�|t |�d kr�| j � d� qd}| j � d� qd S )NF� ro Trj i�� r ) rq �finditerrm �group�startr ZisSurrogatePairZsurrogatePairToCodepoint�non_bmp_invalid_codepointsr[ r7 r( )r% r8 �skip�matchZ codepointr) Zchar_valr r r rR ) s" �z*HTMLUnicodeInputStream.characterErrorsUCS2Fc C s zt ||f }W nh tk rx |D ]}t|�dk s$t�q$d�dd� |D ��}|sZd| }t�d| � }t ||f< Y nX g }|�| j| j �}|dkr�| j | j kr�q�n0|� � }|| j kr�|� | j| j |� � || _ q�|� | j| j d� � | � � s~q�q~d�|�} | S )z� Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its characters. � rX c S s g | ]}d t |� �qS )z\x%02x)rm )r �cr r r r N s z5HTMLUnicodeInputStream.charsUntil.<locals>.<listcomp>z^%sz[%s]+N)�charsUntilRegEx�KeyErrorrm r- r: �re�compilerz r* rZ rY �endr7 rg ) r% Z charactersZopposite�charsr| Zregexr; �mr� �rr r r � charsUntil@ s0 z!HTMLUnicodeInputStream.charsUntilc C sT |d k rP| j dkr.|| j | _| jd7 _n"| j d8 _ | j| j |ksPt�d S r' )rZ r* rY r- )r% rh r r r �ungeto s z HTMLUnicodeInputStream.unget)N)F)r< r= r> r? rl r& rW rU rd r$ rh rg rQ rR r� r� r r r r rF � s & /rF c @ sL e Zd ZdZddd�Zdd� Zd d � Zdd d �Zd d� Zdd� Z dd� Z dS )rG rK N� windows-1252Tc C sn | � |�| _t�| | j� d| _d| _|| _|| _|| _|| _ || _ | � |�| _ | j d dk sbt �| �� dS )rL i �d r N)rU � rawStreamrF r& � numBytesMeta�numBytesChardet�override_encoding�transport_encoding�same_origin_parent_encoding�likely_encoding�default_encoding�determineEncodingrT r- rW )r% rH r� r� r� r� r� Z useChardetr r r r& � s z HTMLBinaryInputStream.__init__c C s& | j d j�| jd�| _t�| � d S )Nr rn )rT Z codec_info� streamreaderr� rV rF rW r6 r r r rW � s zHTMLBinaryInputStream.resetc C sD t |d�r|}nt|�}z|�|�� � W n t|�}Y nX |S r_ )rD r r0 r+ r! r` r r r rU � s z HTMLBinaryInputStream.openStreamc C s� | � � df}|d d k r |S t| j�df}|d d k r:|S t| j�df}|d d k rX|S | �� df}|d d k rt|S t| j�df}|d d k r�|d j�d�s�|S t| j�df}|d d k r�|S |�rpzddl m } W n t k r� Y n�X g }|� }|j �s<| j �| j�}t|t��s t�|�s&�q<|�|� |�|� q�|�� t|jd �}| j �d� |d k �rp|dfS t| j�df}|d d k �r�|S td�dfS )NrN r Z tentativezutf-16)�UniversalDetector�encodingr� )� detectBOMrS r� r� �detectEncodingMetar� �name� startswithr� Z%pip._vendor.chardet.universaldetectorr� � ImportErrorZdoner� r4 r� rB r3 r- r7 Zfeed�close�resultr0 r� )r% ZchardetrT r� ZbuffersZdetectorr# r� r r r r� � sR z'HTMLBinaryInputStream.determineEncodingc C s� | j d dkst�t|�}|d kr&d S |jdkrFtd�}|d k s�t�nT|| j d krf| j d df| _ n4| j�d� |df| _ | �� td| j d |f ��d S )Nr rN ��utf-16be�utf-16lerM r z Encoding changed from %s to %s)rT r- rS r� r� r0 rW r )r% Z newEncodingr r r �changeEncoding s z$HTMLBinaryInputStream.changeEncodingc C s� t jdt jdt jdt jdt jdi}| j�d�}t|t �s<t �|� |dd� �}d}|s~|� |�}d}|s~|� |dd � �}d }|r�| j� |� t |�S | j� d � dS dS ) z�Attempts to detect at BOM at the start of the stream. If an encoding can be determined from the BOM return the name of the encoding otherwise return NonerM r� r� zutf-32lezutf-32be� N� rt r )�codecs�BOM_UTF8� BOM_UTF16_LE� BOM_UTF16_BE� BOM_UTF32_LE� BOM_UTF32_BEr� r4 rB r3 r- �getr0 rS )r% ZbomDict�stringr� r0 r r r r� s4 � z HTMLBinaryInputStream.detectBOMc C sV | j �| j�}t|t�s t�t|�}| j �d� |�� }|dk rR|j dkrRt d�}|S )z9Report the encoding declared by the meta element r Nr� rM ) r� r4 r� rB r3 r- �EncodingParserr0 � getEncodingr� rS )r% r# �parserr� r r r r� 9 s z(HTMLBinaryInputStream.detectEncodingMeta)NNNNr� T)T) r< r= r> r? r& rW rU r� r� r� r� r r r r rG � s � * >"rG c @ s� e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z e e e �Z dd� Ze e�Zefdd�Zdd� Zdd� Zdd� Zd S ) � EncodingBytesz�String-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raisedc C s t |t�st�t�| |�� �S �N)rB r3 r- �__new__�lower�r% �valuer r r r� L s zEncodingBytes.__new__c C s d| _ d S )Nr )rd r� r r r r& P s zEncodingBytes.__init__c C s | S r� r r6 r r r �__iter__T s zEncodingBytes.__iter__c C s> | j d }| _ |t| �kr"t�n |dk r.t�| ||d � S )Nr r �rd r( � StopIterationrE �r% �pr r r �__next__W s zEncodingBytes.__next__c C s | � � S r� )r� r6 r r r �next_ s zEncodingBytes.nextc C sB | j }|t| �krt�n |dk r$t�|d | _ }| ||d � S r' r� r� r r r �previousc s zEncodingBytes.previousc C s | j t| �krt�|| _ d S r� �rd r( r� )r% r$ r r r � setPositionl s zEncodingBytes.setPositionc C s* | j t| �krt�| j dkr"| j S d S d S )Nr r� r6 r r r � getPositionq s zEncodingBytes.getPositionc C s | | j | j d � S �Nr )r$ r6 r r r �getCurrentByte{ s z EncodingBytes.getCurrentBytec C sH | j }|t| �k r>| ||d � }||kr4|| _|S |d7 }q|| _dS )z Skip past a list of charactersr N�r$ r( rd �r% r� r� r| r r r ry � s zEncodingBytes.skipc C sH | j }|t| �k r>| ||d � }||kr4|| _|S |d7 }q|| _d S r� r� r� r r r � skipUntil� s zEncodingBytes.skipUntilc C s> | j }| ||t|� � }|�|�}|r:| j t|�7 _ |S )z�Look for a sequence of bytes at the start of a string. If the bytes are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone)r$ r( r� )r% r3 r� r8 r; r r r � matchBytes� s zEncodingBytes.matchBytesc C sR | | j d� �|�}|dkrJ| jdkr,d| _| j|t|� d 7 _dS t�dS )z�Look for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the matchNr r r T)r$ �findrd r( r� )r% r3 Z newPositionr r r �jumpTo� s zEncodingBytes.jumpToN)r< r= r> r? r� r& r� r� r� r� r� r� �propertyr$ r� � currentByte�spaceCharactersBytesry r� r� r� r r r r r� H s r� c @ sX e Zd ZdZdd� Zdd� Zdd� Zdd � Zd d � Zd d � Z dd� Z dd� Z dd� Z dS )r� z?Mini parser for detecting character encoding from meta elementsc C s t |�| _d| _dS )z3string - the data to work on for encoding detectionN)r� r8 r� �r% r8 r r r r& � s zEncodingParser.__init__c C s� d| j fd| jfd| jfd| jfd| jfd| jff}| jD ]Z}d}|D ]D\}}| j�|�rFz|� }W q�W qF tk r� d}Y q�Y qFX qF|s: q�q:| jS ) Ns <!--s <metas </s <!s <?r TF) � handleComment� handleMeta�handlePossibleEndTag� handleOther�handlePossibleStartTagr8 r� r� r� )r% ZmethodDispatchrs Z keepParsing�key�methodr r r r� � s( � zEncodingParser.getEncodingc C s | j �d�S )zSkip over commentss -->�r8 r� r6 r r r r� � s z EncodingParser.handleCommentc C s� | j jtkrdS d}d }| �� }|d kr,dS |d dkr\|d dk}|r�|d k r�|| _dS q|d dkr�|d }t|�}|d k r�|| _dS q|d dkrtt|d ��}|�� }|d k rt|�}|d k r|r�|| _dS |}qd S ) NTFr s http-equivr s content-type� charsets content) r8 r� r� � getAttributer� rS �ContentAttrParserr� �parse)r% Z hasPragmaZpendingEncoding�attrZtentativeEncoding�codecZ contentParserr r r r� � s8 zEncodingParser.handleMetac C s | � d�S )NF)�handlePossibleTagr6 r r r r� � s z%EncodingParser.handlePossibleStartTagc C s t | j� | �d�S )NT)r� r8 r� r6 r r r r� � s z#EncodingParser.handlePossibleEndTagc C sb | j }|jtkr(|r$|�� | �� dS |�t�}|dkrD|�� n| �� }|d k r^| �� }qLdS )NTr )r8 r� �asciiLettersBytesr� r� r� �spacesAngleBracketsr� )r% ZendTagr8 r| r� r r r r� � s z EncodingParser.handlePossibleTagc C s | j �d�S )Nr r� r6 r r r r� s zEncodingParser.handleOtherc C s� | j }|�ttdg�B �}|dks2t|�dks2t�|dkr>dS g }g }|dkrV|rVq�nX|tkrj|�� }q�nD|dkr�d�|�dfS |tkr�|�|� � � n|dkr�dS |�|� t |�}qF|dkr�|� � d�|�dfS t |� |�� }|dk�rJ|}t |�}||k�r"t |� d�|�d�|�fS |tk�r<|�|� � � q�|�|� q�nJ|d k�rbd�|�dfS |tk�r||�|� � � n|dk�r�dS |�|� t |�}|t k�r�d�|�d�|�fS |tk�r�|�|� � � n|dk�r�dS |�|� �q�dS ) z_Return a name,value pair for the next attribute in the stream, if one is found, or None� /Nr )r N� =)r� r r9 )� '� "r ) r8 ry r� � frozensetr( r- r: �asciiUppercaseBytesr7 r� r� r� r� )r% r8 r| ZattrNameZ attrValueZ quoteCharr r r r� sb zEncodingParser.getAttributeN) r< r= r> r? r& r� r� r� r� r� r� r� r� r r r r r� � s $r� c @ s e Zd Zdd� Zdd� ZdS )r� c C s t |t�st�|| _d S r� )rB r3 r- r8 r� r r r r& f s zContentAttrParser.__init__c C s � z�| j �d� | j jd7 _| j �� | j jdks<W d S | j jd7 _| j �� | j jdkr�| j j}| j jd7 _| j j}| j �|�r�| j || j j� W S W d S nR| j j}z"| j �t� | j || j j� W W S tk � r� | j |d � Y W S X W n tk �r Y d S X d S )Nr� r r� )r� r� )r8 r� r$ ry r� r� r� r� )r% Z quoteMarkZ oldPositionr r r r� j s. zContentAttrParser.parseN)r< r= r> r& r� r r r r r� e s r� c C sf t | t�r0z| �d�} W n tk r. Y dS X | dk r^z t�| �W S tk rZ Y dS X ndS dS )z{Return the python codec name corresponding to an encoding or None if the string doesn't correspond to a valid encoding.r N)rB r �decode�UnicodeDecodeErrorr �lookup�AttributeError)r� r r r rS � s rS )4Z __future__r r r Zpip._vendor.sixr r Zpip._vendor.six.movesr r r� r Z pip._vendorr Z constantsr r r r r rX r �ior r r� r� r� r� r� r� Z invalid_unicode_no_surrogaterO rb r- r� �evalrq �setrx Zascii_punctuation_rer} �objectr! rJ rF rG r3 r� r� r� rS r r r r �<module> s� "�� � J g Ih 6'