o
    䯪g-                     @   s  d dl Z d dlZd dlmZ zd dlmZ d dlmZm	Z	m
Z
 W n ey+   dZY nw zd dlZd dlmZ W n eyE   d ZZY nw e jjZe jjdd Zdd Zdd	 Ze jjd
d Ze jjdd Ze jje jdde gdd Ze jje jjdd Ze jje jdde gdd Ze jje jjdd Ze jje jjdd Z e jje jjdd Z!e jjdd Z"dd Z#dd  Z$dS )!    N)fs)_read_table_test_dataframe_range_integersc           
      C   s   t d}d|d< tjj|dd}t }tj||jdd}g }tdD ]}||d< tjj|dd}|	| |
|  q#|  | }tt|}tj|d	d
}	t| |	 d S Nd   r   	unique_idFpreserve_index2.6version
   Tignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpected r/   e/var/www/html/chatdoc2/venv/lib/python3.10/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_build,   s    
r1   c              	   C   s   t dt  t dt  g}t |}t dgt dgg}t j|ddg}| d }tj	||dddd	+}t
t || W d    n1 sNw   Y  W d    d S W d    d S 1 sfw   Y  d S )
NPOSdesc   blazsimple_validate_schema.parquetr   snappyspark)r   compressionflavor)r   fielduint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r%   simple_fieldssimple_schemasimple_from_arraysimple_tablepathwr/   r/   r0    test_validate_schema_write_tableG   s$   
"rH   c                 C   s   t t ttdt g}td | W d    n1 s"w   Y  t t t| d d  W d    d S 1 s@w   Y  d S )Nx	some_path)	r?   r@   	TypeErrorr   r   r:   int32r   r   )r%   some_schemar/   r/   r0   test_parquet_invalid_writer]   s   "rN   c           
      C   s   t d}d|d< tjj|dd}t }tj||jdd)}g }tdD ]}||d< tjj|dd}|	| |
|  q$W d    n1 sIw   Y  | }tt|}tj|d	d
}	t| |	 d S r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r/   r/   r0   test_parquet_writer_context_objg   s"   

rO   c              
   C   s"  t d}d|d< tjj|dd}t }d}zDtj||jdd1}g }td	D ]"}||d< tjj|dd}|	| |
|  |d
krIt|q'W d    n1 sTw   Y  W n tyt } zt||ksjJ W Y d }~nd }~ww | }	tt|	}
tj|dd}t|
 | d S )Nr   r   r   Fr	   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   rA   	Exceptionstrr   r   r   r   r    r!   r"   r#   )r%   r&   r'   r(   
error_textr)   r*   r+   er,   r-   r.   r/   r/   r0   .test_parquet_writer_context_obj_with_exception   s>   
rU   
filesystemc           	      C   s  t d}tjj|dd}tjj|dd}t| d }t| d }tj||j|dd}|	| W d    n1 s:w   Y  t
| }t|| tj||j|dd}|| W d    n1 sew   Y  t
| }t|| tj||j|dd}|| W d    n1 sw   Y  t
| }t|| tj||j|dd}|| W d    n1 sw   Y  t
| }t|| d S )Nr   Fr	   zdata_table.parquetzdata_batch.parquetr   rV   r   )r   r   r   r   RecordBatchrR   r   r   r   r   r   r#   r!   r"   write_batchwrite)	r%   rV   r&   tablebatch
path_table
path_batchr)   r-   r/   r/   r0   "test_parquet_writer_write_wrappers   sJ   



r_   c                    s   dd  fdd}|d d d |d d |d d d |d d d | d  d d |d d |d d d d S )	Ni   i   c           	         s   t jjt| dgdgd}|d u rt|d  n
tj|d |d td }|d u r1n|}|j|ks:J t| }t	|d D ]}|
|j|ksQJ qE| ||d   }|dkrl|
|d j|ksjJ d S |
|d j|ksxJ d S )NbrI   )namesztest.parquet)row_group_sizer4   r   )r   r   r>   r   r   r   read_metadatanum_row_groupsminr   	row_groupnum_rows)		data_size
chunk_sizeexpect_num_chunksr[   metadataexpected_chunk_sizelatched_chunk_size	chunk_idx	remainderabs_max_chunk_sizedefault_chunk_sizer%   r/   r0   check_chunk_size   s,   
z8test_parquet_writer_chunk_size.<locals>.check_chunk_size   r      r4   r/   )r%   rs   r/   rp   r0   test_parquet_writer_chunk_size   s   rv   c                 C   s~   t d}tjj|dd}t| d }tj||j|dd}|| W d    n1 s,w   Y  t	|
 }t|| d S )Nr   Fr	   zdata.parquetr   rW   )r   r   r   r   rR   r   r   r   r   r   r#   r!   r"   )r%   rV   r&   r[   rF   r)   r-   r/   r/   r0   $test_parquet_writer_filesystem_local   s   
rw   c                 C   s|   t d}tjj|dd}| \}}}tj||j|dd}|| W d    n1 s+w   Y  t|	 }t
|| d S )Nr   Fr	   r   rW   r   r   r   r   r   r   r   r   r   r#   r!   r"   s3_example_fsr&   r[   r   urirF   r)   r-   r/   r/   r0   !test_parquet_writer_filesystem_s3  s   

r|   c                 C   s~   t d}tjj|dd}| \}}}tj||jdd}|| W d    n1 s*w   Y  t||d	 }t
|| d S )Nr   Fr	   r   r   rV   rx   ry   r/   r/   r0   %test_parquet_writer_filesystem_s3_uri  s   
r~   c                 C   s   t d}tjj|dd}| \}}|d }tj||j|dd}|| W d    n1 s.w   Y  t||d	 }t
|| d S )Nr   Fr	   z/test.parquetr   rW   r}   rx   )s3_example_s3fsr&   r[   r   	directoryrF   r)   r-   r/   r/   r0   #test_parquet_writer_filesystem_s3fs+  s   
r   c                  C   sj   t d} tjj| dd}t }tjtdd t	j
t |j|d W d    d S 1 s.w   Y  d S )Nr   Fr	   zspecified path is file-like)matchr}   )r   r   r   r   r   LocalFileSystemr?   r@   rA   r   r   r   r   )r&   r[   rV   r/   r/   r0   ,test_parquet_writer_filesystem_buffer_raises=  s   "r   c                 C   s   t dg di}| d }t||j}|| W d    n1 s$w   Y  t|}d|jv s5J |jd s<J | d }tj||jdd}|| W d    n1 sYw   Y  t|}|jd u sjJ d S )Na)r4   rt   ru   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)store_schema)r   r[   r   r   r   r   rc   rk   )r%   r[   path1r)   metapath2r/   r/   r0    test_parquet_writer_store_schemaJ  s   

r   c                 C   s   t jt jg ddgdg}| d }t||j}|| |ddd |dd	d
 W d    n1 s9w   Y  t	|}|j
j
}|d dksOJ |d dksWJ |d dks_J d S )NrL   )typef0zmetadata.parquet1rI   )key1key223)r   key3s   key1   1s   key2   2s   key3   3)r   r   r>   r=   r   r   r   r   add_key_value_metadataParquetFilerk   )r%   r[   rF   r)   readerrk   r/   r/   r0   -test_parquet_writer_append_key_value_metadata_  s   

r   )%r?   pyarrowr   r   pyarrow.parquetparquetr   pyarrow.tests.parquet.commonr   r   r   ImportErrorpandasr   pandas.testingtestingr!   mark
pytestmarkr1   rH   rN   rO   rU   parametrizer   r_   large_memoryrv   rw   s3r|   r~   r   r   r   r   r/   r/   r/   r0   <module>   sh   



()
