o
    䯪g&                     @   s  d dl Z d dlZd dlZd dlZd dlZzd dlmZ d dl	m
Z
 W n ey-   dZY nw zd dlZd dlmZ d dl	mZ W n eyM   d ZZY nw ejjZejjdd Zejjdd Zejjdd	 Zejjd
d Zejjdd Zejjdd Zdd Zdd Zejjejdg ddd Zejjejddgdd Zejjejdddgdd  Zd!d" Z ejj!ejd#d$d%d& Z"dS )'    N)_write_table)alltypes_samplec                  C   sr   t dd} tj| }t }t||ddd |d t	|}|d tj
||d}t| |   d S )N'  sizesnappy2.6)compressionversionr   )metadata)r   paTablefrom_pandasioBytesIOr   seekpqread_metadataParquetFiletmassert_frame_equalread	to_pandas)dfa_tablebufr   fileh r   c/var/www/html/chatdoc2/venv/lib/python3.10/site-packages/pyarrow/tests/parquet/test_parquet_file.pytest_pass_separate_metadata.   s   



r   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|  j
|ks/J  fddt|D }t|}t||  d S )	Nr      r   r   r   row_group_sizer	   r
   r   c                    s   g | ]}  |qS r   read_row_group.0ipfr   r   
<listcomp>T   s    z.test_read_single_row_group.<locals>.<listcomp>)r   r   r   r   r   r   r   r   r   r   num_row_groupsrangeconcat_tablesr   r   r   NKr   r   r   
row_groupsresultr   r)   r   test_read_single_row_groupB   s   



r4   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|t
|jd d   fdd	t|D }t|}t|  |   fd
d	t|D }t|}t|  |  d S )Nr    r   r   r   r"   r      c                    s   g | ]	}j | d qS columnsr$   r&   colsr*   r   r   r+   g   s    zAtest_read_single_row_group_with_column_subset.<locals>.<listcomp>c                    s   g | ]}j |   d qS r6   r$   r&   r9   r   r   r+   m   s    )r   r   r   r   r   r   r   r   r   r   listr8   r-   r.   r   r   r   r/   r   r9   r   -test_read_single_row_group_with_column_subsetY   s    




r<   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|j
|ks/J |t|}t||  d S )Nr    r   r   r   r"   r   )r   r   r   r   r   r   r   r   r   r   r,   read_row_groupsr-   r   r   r   )r0   r1   r   r   r   r*   r3   r   r   r   test_read_multiple_row_groupsr   s   


r>   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}t
|jd d }|jt||d}t|| |  |jt||| d}t|| |  d S )	Nr    r   r   r   r"   r   r5   r7   )r   r   r   r   r   r   r   r   r   r   r;   r8   r=   r-   r   r   r   )r0   r1   r   r   r   r*   r:   r3   r   r   r   0test_read_multiple_row_groups_with_column_subset   s   


r?   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|
 dks0J |
|jd d dks>J d S )	Nr    r   r   r   r"   r   r   r!   )r   r   r   r   r   r   r   r   r   r   scan_contentsr8   )r0   r1   r   r   r   r*   r   r   r   test_scan_contents   s   


 rA   c                 C   s   | d }t t| dt| d}tt}t| W d    n1 s(w   Y  |t	r9t
jdkr9d S || d S )N	directoryzCannot open for reading: path 'z' is a directorywin32)osmkdirstrpytestraisesIOErrorr   r   errisinstancePermissionErrorsysplatformmatch)tempdirpathmsgexcr   r   r   0test_parquet_file_pass_directory_instead_of_file   s   rS   c               	   C   s   t jt ddgt ddggddgd} t  }t| | t| }|j	d
 ddgks4J |j	d	
 ddgksBJ d
D ]}tttf |j	| W d    n1 s^w   Y  qDd S )Nr!      foobarintsstrs)namesr      )r5   )r   tablearrayBufferOutputStreamr   write_tabler   getvaluereaderread_column	to_pylistrG   rH   
ValueError
IndexError)r\   biofindexr   r   r   test_read_column_invalid_index   s   ri   
batch_size)i,    i  c              	   C   s   d}d}t |d}| d }tj|}t||d|d t|}|jd d |jdd  fD ]>}|j||d}	t	d	|| |}
t
|	|
D ]'\}}t||| }t| |j||d d f jd d |f jd
d qDq.d S )Ni  rk   r   pandas_roundtrip.parquetr   r
   
chunk_size
   )rj   r8   r   Tdrop)r   r   r   r   r   r   r   r8   iter_batchesr-   zipminr   r   r   iloclocreset_index)rO   rj   
total_sizern   r   filenamearrow_tablefile_r8   batchesbatch_startsbatchstartendr   r   r    test_iter_batches_columns_reader   s(   

 *r   rn   rk   c           
      C   s   t ddd}| d }tj|}|jjd usJ t||d|d t|}dd }t	||}d	}t
|jD ]:}	t||  ||	g d
 |d7 }t||  jdd||	g jd
d  jdd |d7 }q6d S )Nr   T)r   categoricalrl   r   rm   c                 s   s6    t | jD ]}| jd|gd}|D ]}|V  qqd S )N  )rj   r2   )r-   r,   rr   )rg   	row_groupr|   r~   r   r   r   get_all_batches   s   z1test_iter_batches_reader.<locals>.get_all_batchesr   r   rZ   rp   )r   r   r   r   schemapandas_metadatar   r   r   r;   r-   r,   r   r   r   r=   headrw   ru   )
rO   rn   r   ry   rz   r{   r   r|   batch_nor(   r   r   r   test_iter_batches_reader   s2   



r   
pre_bufferFTc                 C   sj   d\}}t |d}tj|}t }t|||| ddd |d tj	|| d}|
 j|ks3J d S )Nr    r   r   r   r"   r   )r   )r   r   r   r   r   r   r   r   r   r   r   num_rows)r   r0   r1   r   r   r   r*   r   r   r   test_pre_buffer  s   

r   c              	   C   s  |  d}tddgddgd}t|| t|d5}t|}|  |jr,J |jr1J W d   n1 s;w   Y  |jrEJ |jrJJ W d   n1 sTw   Y  |js^J |jscJ t|}|  |jrrJ W d   n1 s|w   Y  |jsJ dS )z
    Unopened files should be closed explicitly after use,
    and previously opened files should be left open.
    Applies to read_table, ParquetDataset, and ParquetFile
    zfile.parquetr   rZ   )col1col2rbN)	joinpathr   r\   r   r_   openr   r   closed)rO   fnr\   rg   pr   r   r   #test_parquet_file_explicitly_closed  s&   




r   use_uri)TFc           
      C   s   | \}}}|r	|n|f}|ri nt |d}tdtdi}tj|||d tj|i |}| |ks6J |jr;J |	  |jsDJ tj|i |}	|	 |ksUJ |	jrZJ W d    n1 sdw   Y  |	jsnJ d S )N)
filesystemaro   )
dictr   r\   r-   r   r_   r   r   r   close)
s3_example_fsr   s3_fss3_uris3_pathargskwargsr\   parquet_filerg   r   r   r   !test_parquet_file_with_filesystem<  s   


r   )#r   rD   rL   rG   pyarrowr   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   ImportErrorpandaspdpandas.testingtestingr   r   mark
pytestmarkr   r4   r<   r>   r?   rA   rS   ri   parametrizer   r   r   r   s3r   r   r   r   r   <module>   s\   





+