B
    /b                 @   s,  d Z ddlmZ ddlZddlZddlZddlZddlmZ ddlm	Z	m
Z
 ddlmZmZ edd	Zejd
dddd ejdddd ejddddd ejdddddd ejddddd d ejd!d"d#d ejd$d%dd&d ejddd'd(d(d(d) d*gZd+d, Zd-d. Zd/d0 Zed1kr(e  dS )2zJwarc2warc - convert one warc to another, can be used to re-compress things    )print_functionN)OptionParser   )
WarcRecordexpand_files)RequestMessageResponseMessagez%prog [options] url (url ...))usagez-oz--outputoutputzoutput warc file)desthelpz-lz--limitlimit)r   z-Iz--inputZinput_formatz	(ignored)z-Zz--gzipgzip
store_truez!compress output, record by record)r   actionr   z-Dz--decode_httpdecode_httpz)decode http messages (strip chunks, gzip)z-Lz--log-level	log_levelz--wget-chunk-fixwget_workaroundzGskip transfer-encoding headers in http records, when decoding them (-D)infoF)Zoutput_directoryr   r   r   r   r   zTransfer-Encodingc       	      C   s   |j r
tnd}|jr| jtjkr| j\}}d }|tjkrFtt	 |d}|t	jkrZt	|d}|r|
|}|  |s| r| }||f| _nTg }|r|dt|  | s|d|j|jjf  td| jd|tjd | j||jd d S )	N )ignore_headersz%d bytes unparsedzincomplete message (at %s, %s)zerrors decoding http in record,)file)r   )r   WGET_IGNORE_HEADERSr   typer   ZRESPONSEcontentr   CONTENT_TYPEr   feedcloseZcompleteZget_decoded_messageappendlenmodeheaderprintidjoinsysstderrZwrite_tor   )	recordoutoptionsr   content_typer   messageleftovererrorr   r   3/usr/lib/python3.7/site-packages/hanzo/warc2warc.pyprocess!   s,    




r0   c             C   s   t j| dd  d\}}ytjj}W n tk
r>   tj}Y nX t|dk rxtjtj	d d}x\|D ]}t
||| qbW nBx@t|D ]4}tj|dd}x|D ]}t
||| qW |  qW dS )Nr   )args)Zfile_handler   auto)r   r   )parser
parse_argsr&   stdoutbufferAttributeErrorr    r   Zopen_archivestdinr0   r   r   )argvr*   Zinput_filesr)   fhr(   namer   r   r/   main?   s    

r<   c               C   s   t tt j d S )N)r&   exitr<   r9   r   r   r   r/   runX   s    r>   __main__)__doc__
__future__r   osr&   os.pathoptparser   Z	warctoolsr   r   Z	httptoolsr   r   r3   
add_optionset_defaultsr   r0   r<   r>   __name__r   r   r   r/   <module>   s0   


