
    gSV                     >   d dl Z d dlmZ d dlZ	 d dlmZ d dlmc m	Z
 d dlmZmZ dZdZdZdZdZe j(                  j*                  e j(                  j                  gZ e j.                  d	
      d        Z e j.                  d	
      d        Zd Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*e j(                  jW                  d      d         Z,e j(                  jW                  d!      d"        Z-d# Z.d$ Z/d% Z0y# e$ r dZdZ
Y w xY w)&    N)	timedelta)InMemoryKmsClientverify_file_encryptedzencrypted_table.in_mem.parquets   0123456789112345
footer_keys   1234567890123450col_keymodule)scopec                      t         j                  j                  t        j                  g d      t        j                  g d      t        j                  g d      d      } | S )N)         )abc)xyz)paTablefrom_pydictarray)
data_tables    Z/var/www/openai/venv/lib/python3.12/site-packages/pyarrow/tests/parquet/test_encryption.pyr   r   0   sG    %%XXi XXo&XXo&' J
     c                  L    t        j                  t        t        ddgi      } | S )Nr   r   )r   column_keys)peEncryptionConfigurationFOOTER_KEY_NAMECOL_KEY_NAME)basic_encryption_configs    r   r!   r!   :   s,     88"3*

 #"r   c                 f    t        j                  |       }d }t        j                  |      }||fS )z
    Sets up and returns the KMS connection configuration and crypto factory
    based on provided KMS configuration parameters.
    custom_kms_confc                     t        |       S Nr   kms_connection_configurations    r   kms_factoryz1setup_encryption_environment.<locals>.kms_factoryK        !=>>r   )r   KmsConnectionConfigCryptoFactory)r$   kms_connection_configr*   crypto_factorys       r   setup_encryption_environmentr0   D   s7    
 22?S? %%k2N .00r   c                     ||j                  d      ||j                  d      i}t        |      \  }}	t        | ||||	       ||	fS )zL
    Writes an encrypted parquet file based on the provided parameters.
    UTF-8)decoder0   write_encrypted_parquet)
pathr   footer_key_namecol_key_namer   r   encryption_configr$   r.   r/   s
             r   write_encrypted_filer9   T   sc     	**73gnnW-O -I-)> D*.?1>C !.00r   c           	      V   | t         z  }t        j                  t        t        ddgidt        d      d      }t        ||t        t        t        t        |      \  }}t        |       t        j                  t        d            }t        ||||      }|j                  |      sJ y	)
zDWrite an encrypted parquet, verify it's encrypted, and then read it.r   r   
AES_GCM_V1      @minutes   r   r   encryption_algorithmcache_lifetimedata_key_length_bitsrB   N)PARQUET_NAMEr   r   r   r    r   r9   
FOOTER_KEYCOL_KEYr   DecryptionConfigurationread_encrypted_parquetequals)tempdirr   r5   r8   r.   r/   decryption_configresult_tables           r   !test_encrypted_parquet_write_readrN   j   s    \!D
 22"3*
 * - " -Aj/<W-)> $ 22 -/)!6HL\***r   c                     |j                  ||      }|J t        j                  | |j                  |      5 }|j	                  |       d d d        y # 1 sw Y   y xY w)N)encryption_properties)file_encryption_propertiespqParquetWriterschemawrite_table)r5   tabler8   r.   r/   rQ   writers          r   r4   r4      s`    !/!J!J0"2%111			%,,"<
>AG5!
> 
> 
>s   AAc                 2   |j                  ||      }|J t        j                  | |      }|j                  dk(  sJ t        j                  | |      }t        |j                        dk(  sJ t        j                  | |      }|j                  d      S )Ndecryption_propertiesr   Tuse_threads)	file_decryption_propertiesrR   read_metadatanum_columnsread_schemalennamesParquetFileread)r5   rL   r.   r/   r]   metarT   results           r   rI   rI      s    !/!J!J0"2%111$>@Dq   ^^$>@Fv||!!!^^$>@F;;4;((r   c           	      
   | t         z  }t        j                  t        t        ddgidt        d      d      }t        ||t        t        t        t        |       t        |       t        t        t        j                  d      t        t        j                  d      i      \  }}t        j                  t        d      	      }t        j                  t        d
      5  t!        ||||       ddd       y# 1 sw Y   yxY w)zYWrite an encrypted parquet, verify it's encrypted,
    and then read it using wrong keys.r   r   r;   r<   r=   r?   r@   r2   rD   zIncorrect master key usedmatchN)rE   r   r   r   r    r   r9   rF   rG   r   r0   r3   rH   pytestraises
ValueErrorrI   )rK   r   r5   r8   wrong_kms_connection_configwrong_crypto_factoryrL   s          r   +test_encrypted_parquet_write_read_wrong_keyro      s     \!D
 22"3*
 * - " z?L#W.?A $8T0j''0V 95!5
 22 -/	z)E	F#%@ 	" 
G	F	Fs   !C99Dc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z        j                          ddd       y# 1 sw Y   yxY w)zmWrite an encrypted parquet, verify it's encrypted,
    but then try to read it without decryption properties.no decryptionrh   N)rN   rj   rk   IOErrorrR   rc   rE   rd   rK   r   s     r   0test_encrypted_parquet_read_no_decryption_configrt      sC     &gz:	w&6	7
w-.335 
8	7	7s   +AA%c                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zwWrite an encrypted parquet, verify it's encrypted,
    but then try to read its metadata without decryption properties.rq   rh   N)rN   rj   rk   rr   rR   r^   rE   rs   s     r   9test_encrypted_parquet_read_metadata_no_decryption_configrv      s<     &gz:	w&6	7
</0 
8	7	7   AAc                     t        | |       t        j                  t        d      5  t	        j
                  | t        z         ddd       y# 1 sw Y   yxY w)zuWrite an encrypted parquet, verify it's encrypted,
    but then try to read its schema without decryption properties.rq   rh   N)rN   rj   rk   rr   rR   r`   rE   rs   s     r   7test_encrypted_parquet_read_schema_no_decryption_configry      s:     &gz:	w&6	7
w-. 
8	7	7rw   c           
          | dz  }t        j                  t              }t        j                  t
        d      5  t        ||t        t        t        d|       ddd       y# 1 sw Y   yxY w)zMWrite an encrypted parquet, but give only footer key,
    without column key.z)encrypted_table_no_col_key.in_mem.parquetr   z4Either column_keys or uniform_encryption must be setrh   r   N)	r   r   r   rj   rk   OSErrorr9   r    rF   )rK   r   r5   r8   s       r   'test_encrypted_parquet_write_no_col_keyr}      se     @@D 22"$ 
w%
& 	T:'.?	A	
& 
& 
&s   A""A+c                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)<Write an encrypted parquet, but raise KeyError in KmsClient.(encrypted_table_kms_error.in_mem.parquetc                     t        |       S r&   r'   r(   s    r   r*   z;test_encrypted_parquet_write_kms_error.<locals>.kms_factory   s     !!=>>r   r   rh   N)r   r,   r-   rj   rk   KeyErrorr4   rK   r   r!   r5   r8   r.   r*   r/   s           r   &test_encrypted_parquet_write_kms_errorr      sl     ??D/ 224?
 %%k2N	x|	4j2C 5~	G 
5	4	4   A((A1c                     | dz  }|}t        j                         } G d dt         j                        fd}t        j                  |      }t	        j
                  t        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)r   r   c                   "    e Zd ZdZd Zd Zd Zy)Jtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClientzVA KmsClient implementation that throws exception in
        wrap/unwrap calls
        c                 P    t         j                  j                  |        || _        y)z%Create an InMemoryKmsClient instance.N)r   	KmsClient__init__configselfr   s     r   r   zStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.__init__  s    LL!!$' DKr   c                     t        d      )NCannot Wrap Keyrl   r   	key_bytesmaster_key_identifiers      r   wrap_keyzStest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.wrap_key  s    .//r   c                     t        d      )NzCannot Unwrap Keyr   r   wrapped_keyr   s      r   
unwrap_keyzUtest_encrypted_parquet_write_kms_specific_error.<locals>.ThrowingKmsClient.unwrap_key  s    011r   N__name__
__module____qualname____doc__r   r   r    r   r   ThrowingKmsClientr     s    		!
	0	2r   r   c                      |       S r&   r   )r)   r   s    r   r*   zDtest_encrypted_parquet_write_kms_specific_error.<locals>.kms_factory"  s     !=>>r   r   rh   N)r   r,   r   r-   rj   rk   rl   r4   )	rK   r   r!   r5   r8   r.   r*   r/   r   s	           @r   /test_encrypted_parquet_write_kms_specific_errorr   	  s~     ??D/ 2242BLL 2 ? %%k2N	z):	;j2C 5~	G 
<	;	;s   +BBc                     | dz  }|}t        j                         }d }t        j                  |      }t        j                  t
        d      5  t        |||||       ddd       y# 1 sw Y   yxY w)z@Write an encrypted parquet, but raise ValueError in kms_factory.0encrypted_table_kms_factory_error.in_mem.parquetc                     t        d      )NCannot create KmsClientr   r(   s    r   r*   zCtest_encrypted_parquet_write_kms_factory_error.<locals>.kms_factory6  s    233r   r   rh   N)r   r,   r-   rj   rk   rl   r4   r   s           r   .test_encrypted_parquet_write_kms_factory_errorr   -  su     GGD/ 2244 %%k2N	z6
8 	 j2C 5~	G
8 
8 
8r   c                     | dz  }|}t        j                         } G d d      fd}t        j                  |      }t        j                  t
              5  t        |||||       ddd       y# 1 sw Y   yxY w)z_Write an encrypted parquet, but use wrong KMS client type
    that doesn't implement KmsClient.r   c                   "    e Zd ZdZd Zd Zd Zy)Otest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClientz4This is not an implementation of KmsClient.
        c                 &    |j                   | _        y r&   )r$   master_keys_mapr   s     r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.__init__O  s    #)#9#9D r   c                      y r&   r   r   s      r   r   zXtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.wrap_keyR      r   c                      y r&   r   r   s      r   r   zZtest_encrypted_parquet_write_kms_factory_type_error.<locals>.WrongTypeKmsClient.unwrap_keyU  r   r   Nr   r   r   r   WrongTypeKmsClientr   K  s    		:		r   r   c                      |       S r&   r   )r)   r   s    r   r*   zHtest_encrypted_parquet_write_kms_factory_type_error.<locals>.kms_factoryX  s    !">??r   N)r   r,   r-   rj   rk   	TypeErrorr4   )	rK   r   r!   r5   r8   r.   r*   r/   r   s	           @r   3test_encrypted_parquet_write_kms_factory_type_errorr   A  sv     GGD/ 224 @ %%k2N	y	!j2C 5~	G 
"	!	!s   A33A<c            
      J   d } t        j                  t        t        ddgidddt	        d      dd	
      } | |       t        j                  t              }t        ddgi|_        d|_        d|_        d|_        t	        d      |_	        d|_
        d	|_         | |       y )Nc                 0   t         | j                  k(  sJ ddg| j                  t           k(  sJ d| j                  k(  sJ | j
                  sJ | j                  rJ t        d      | j                  k(  sJ | j                  rJ d| j                  k(  sJ y )Nr   r   AES_GCM_CTR_V1      $@r=      )r   r   r   r    rA   plaintext_footerdouble_wrappingr   rB   internal_key_materialrC   )r8   s    r   !validate_encryption_configurationzZtest_encrypted_parquet_encryption_configuration.<locals>.validate_encryption_configurationc  s    "3">">>>>Sz.::<HHHH#4#I#IIII 1111$4444&*;*J*JJJJ$::::'<<<<<r   r   r   r   TFr   r=   r   )r   r   rA   r   r   rB   r   rC   r{   )r   r   r   r    r   r   rA   r   r   rB   r   rC   )r   r8   encryption_config_1s      r   /test_encrypted_parquet_encryption_configurationr   b  s    = 22"!C:0- .# 	 &&7844"$'3c3Z&B#/?,+/(*/')24)@&05-/2,%&9:r   c                      t        j                  t        d            } t        d      | j                  k(  sJ t        j                         }t        d      |_        t        d      |j                  k(  sJ y )Nr   r=   rD   )r   rH   r   rB   )rL   decryption_config_1s     r   /test_encrypted_parquet_decryption_configurationr     sl    22 .0T"&7&F&FFFF446)24)@&T"&9&H&HHHHr   c                      d } t        j                  dddddd      } | |       t        j                         }d|_        d|_        d|_        ddd|_         | |       y )	Nc                     d| j                   k(  sJ d| j                  k(  sJ d| j                  k(  sJ ddd| j                  k(  sJ y )N	Instance1URL1MyTokenkey_material_1key_material_2key1key2kms_instance_idkms_instance_urlkey_access_tokenr$   )r.   s    r   validate_kms_connection_configzPtest_encrypted_parquet_kms_configuration.<locals>.validate_kms_connection_config  sb    3CCCCC.?????1BBBBB)3CD%556 	7 6r   r   r   r   r   r   r   r   )r   r,   r   r   r   r$   )r   r.   kms_connection_config_1s      r   (test_encrypted_parquet_kms_configurationr     s    7 22#"$$
	 ##89 446.9+/5,/8,  /+ ##:;r   zNPlaintext footer - reading plaintext column subset reads encrypted columns too)reasonc                 >   | t         z  }t        j                  t        t        ddgidd      }t        j
                  t        t        j                  d      t        t        j                  d      i      }d }t        j                  |      }t        |||||       y	)
zWrite an encrypted parquet, with plaintext footer
    and with single wrapping,
    verify it's encrypted, and then read plaintext columns.r   r   TF)r   r   r   r   r2   r#   c                     t        |       S r&   r'   r(   s    r   r*   zStest_encrypted_parquet_write_read_plain_footer_single_wrapping.<locals>.kms_factory  r+   r   N)rE   r   r   r   r    r,   rF   r3   rG   r-   r4   rK   r   r5   r8   r.   r*   r/   s          r   >test_encrypted_parquet_write_read_plain_footer_single_wrappingr     s     \!D
 22"3*
  22Z..w7'..1
? %%k2ND*.?1>Cr   z'External key material not supported yetc                     | t         z  }t        j                  t        i d      }t        j                  t        t
        j                  d      i      }d }t        j                  |      }t        |||||       y)zWrite an encrypted parquet, with external key
    material.
    Currently it's not implemented, so should throw
    an exceptionF)r   r   r   r2   r#   c                     t        |       S r&   r'   r(   s    r   r*   z:test_encrypted_parquet_write_external.<locals>.kms_factory  r+   r   N)	rE   r   r   r   r,   rF   r3   r-   r4   r   s          r   %test_encrypted_parquet_write_externalr     s|     \!D 22"#%
 22(**;*;G*DE? %%k2ND*.?1>Cr   c           	      x   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }t        d      D ]T  }|j                  ||      }|J t        j                  ||      }	|	j                  d      }
|j                  |
      rTJ  y)	z`Write an encrypted parquet, verify it's encrypted,
    and then read it multithreaded in a loop.r<   r=   rD   2   NrY   Tr[   )rE   r9   r   r    rF   rG   r   r   rH   r   ranger]   rR   rc   rd   rJ   )rK   r   r!   r5   r.   r/   rL   ir]   rf   rM   s              r   test_encrypted_parquet_loopr     s     \!D
 -Aj/<W-!)> $22 -/ 2Y%3%N%N!#4&6")555(BD{{t{4  ... r   c           	      R   | t         z  }t        ||t        t        t        t
        |      \  }}t        |       t        j                  t        d            }|j                  ||      }~t        j                  ||      }|j                  d      }	|j                  |	      sJ y)z^
    Test that decryption properties can be used if the crypto factory is no longer alive
    r<   r=   rD   rY   Tr[   N)rE   r9   r   r    rF   rG   r   r   rH   r   r]   rR   rc   rd   rJ   )
rK   r   r!   r5   r.   r/   rL   r]   rf   rM   s
             r   %test_read_with_deleted_crypto_factoryr     s     \!D,@j/<W-!)> $ 22 -/!/!J!J0"2^^$>@F;;4;0L\***r   c           	      j   | t         z  }t        ||t        t        t        t
        |      \  }}t        j                  t        d            }|j                  ||      }t        j                  ||      }|j                  |      sJ t        j                  | |      }|j                  |      sJ y)z>Write an encrypted parquet then read it back using read_table.r<   r=   rD   rY   N)rE   r9   r   r    rF   rG   r   rH   r   r]   rR   
read_tablerJ   )	rK   r   r!   r5   r.   r/   rL   r]   rM   s	            r   !test_encrypted_parquet_read_tabler   %  s    \!D -Aj/<W-!)> 22 -/!/!J!J0"2 ===WXL \*** =='ACL\***r   )1rj   datetimer   pyarrowr   pyarrow.parquetparquetrR   pyarrow.parquet.encryption
encryptionr    pyarrow.tests.parquet.encryptionr   r   ImportErrorrE   rF   r   rG   r    markparquet_encryption
pytestmarkfixturer   r!   r0   r9   rN   r4   rI   ro   rt   rv   ry   r}   r   r   r   r   r   r   r   xfailr   r   r   r   r   r   r   r   <module>r      s  "   2 ++
2 0 

 KK""
KK
 h   h#  #1 1,+<")" "F61/A"G*!GHG(GB ;FI<:  2 3C3CN CDC EC4/:+0+[  	B	Bs   D 	DD