
    kh                      "   d Z ddlZddlZddlmZmZ ddlmZ ddlZddl	m
Z ddlmZ 	 ddlZddlmZ ddlmZ d	d
lmZ d	dlmZ d	dlmZ  eej:                  j=                  dd            Z G d de      Z y# e$ rZ ed       edZ[ww xY w)z' Dataset reader for HF IterableDataset
    N)repeatchain)Optional)Image)split_dataset_by_node)	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.   )load_class_map)Reader)SharedCountHFIDS_SHUFFLE_SIZEi   c            !            e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedee   dedededededed	ee   d
ededededee   dee   def  fdZd Z		 ddee   fdZ
d Zd Zd Zd ZddZddZ xZS )ReaderHfidsnamerootsplitis_training
batch_sizedownloadrepeatsseed	class_map	input_keyinput_img_mode
target_keytarget_img_modeshuffle_sizenum_samplestrust_remote_codec                    t         |           || _        || _        || _        || _        || _        || _        || _        |xs t        | _
        |
| _        || _        || _        || _        t        j                   |||      | _        |r| j"                  j%                          d }| j"                  j&                  j(                  rv|| j"                  j&                  j(                  v rTt+        | j"                  j&                  j(                  |   t,              r#| j"                  j&                  j(                  |   }|r|| _        n+|r|j0                  r|j0                  | _        nt3        d      d| _        |	rt7        |	      | _        d| _        ni | _        d| _        d| _        t?        j@                         r]t?        jB                         rIt?        jD                         dkD  r2t?        jF                         | _        t?        jD                         | _        d | _$        d| _%        d| _&        d| _'        d| _(        d | _)        tU               | _+        y )N)	cache_dirr   zDataset length is unknown, please pass `num_samples` explicitly. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r	   ),super__init__r   r   r   r   r   r   common_seedSHUFFLE_SIZEr   r   r   r   r   datasetsload_dataset_builderbuilderdownload_and_prepareinfosplits
isinstancer   r   num_examples
ValueErrorremap_classr
   class_to_idx	dist_rankdist_num_replicasdistis_availableis_initializedget_world_sizeget_rankworker_info	worker_idnum_workersglobal_worker_idglobal_num_workersdsr   epoch)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
split_info	__class__s                     Z/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/data/readers/reader_hfids.pyr#   zReaderHfids.__init__   s   & 		
&$ (8L",$.44/

 LL--/*.
<<##1B1B1I1I(I$,,++22599E26,,2C2C2J2J52Q
*DJ33)66Dd 
 ! .y 9D#D "D !"4#6#6#8T=P=P=RUV=V!]]_DN%)%8%8%:D"   !"# 7; ]
    c                 &    || j                   _        y N)r>   value)r?   counts     rB   	set_epochzReaderHfids.set_epochp   s     

rC   r:   c                 n    | j                   y |&|| _        | j                  | j                  z  | _        y y rE   )r=   r:   r2   r<   )r?   r:   s     rB   set_loader_cfgzReaderHfids.set_loader_cfgt   s=     77"*D&*&<&<t?O?O&OD# #rC   c                    | j                   t        j                  j                  j	                         }|)|| _         |j
                  | _        |j                  | _        | j                  | j                  z  | _	        | j                  | j                  z  | j                  z   | _        | j                  rC| j                  j                  | j                        }|j!                  | j                        }n&| j                  j#                  | j                        }| j$                  r'|j'                  | j(                  | j*                        }t-        || j                  | j                        | _        y)z8 Lazily initialize worker (in worker processes)
        N)r   )
num_shards)r   buffer_size)rank
world_size)r8   torchutilsdataget_worker_infoidr9   r:   r2   r<   r1   r;   r   r(   
as_datasetr   to_iterable_datasetas_streaming_datasetr   shuffler$   r   r   r=   )r?   r8   datasetr=   s       rB   
_lazy_initzReaderHfids._lazy_init~   s#    #++**::<K&#. !,#.#:#: &*&<&<t?O?O&OD#$(NNT5E5E$E$VD!==ll--DJJ-?G,,8O8O,PB 222DB!1!1t?P?PQB (DLbLbcrC   c                    t        d| j                        | j                  z  t        | j                  | j                        z  }| j
                  s| j                  dkD  rt        j                  |      }| j
                  r;| j                  /t        j                  || j                  z        | j                  z  }t        |      S )Nr	   )
maxr   r   r<   r2   r   mathceilr   int)r?   num_worker_sampless     rB   _num_samples_per_workerz#ReaderHfids._num_samples_per_worker   s    4<< 4#3#33c$:Q:QSWSiSi6jj 	t559!%+=!> ;!%+=+O!PSWSbSb!b%&&rC   c              #   \  K   | j                   | j                          | j                   j                  | j                  j                         | j                         }d}| j                  r)t        j                  t        | j                               }nt        | j                         }|D ]  }|| j                     }| j                  r4|j                  | j                  k7  r|j                  | j                        }|| j                     }| j                   rVt#        |t$        j$                        sJ d       |j                  | j                   k7  r7|j                  | j                         }n| j&                  r| j(                  |   }||f |dz  }| j                  s||k\  s y  y w)Nr   z7target_img_mode is specified but target is not an imager	   )r=   rZ   rH   r>   rF   ra   r   r   from_iterabler   iterr   r   modeconvertr   r   r,   r   r/   r0   )r?   target_sample_countsample_countds_itersample
input_datatarget_datas          rB   __iter__zReaderHfids.__iter__   sa    77?OO$****+"::<))&/:G477mG 	F&,T^^&<J""z$:M:M'M'//0C0CD
 1K##!+u{{;v=vv;##t';';;"-"5"5d6J6J"KK!!"//<k))ALL4G$G	s   FF,"F,(F,c                 @    | j                         | j                  z  }|S rE   )ra   r:   )r?   r   s     rB   __len__zReaderHfids.__len__   s!    224t7G7GGrC   c                     J d       )NzNot supported )r?   indexbasenameabsolutes       rB   	_filenamezReaderHfids._filename   s    %o%urC   c                     | j                   | j                          g }| j                   D ]B  }d|v r|d   }n%d|v r|d   }nd|v r|d   }nd|v r|d   }nJ d       |j                  |       D |S )z0 Return all filenames in dataset, overrides base	file_namefilenamerT   image_idzNo supported name field present)r=   rZ   append)r?   rs   rt   namesrj   r   s         rB   	filenameszReaderHfids.filenames   s    77?OOgg 	Ff$k*v%j)d|v%j)???uLL	 rC   )NtrainFr	   Fr   *   NimageRGBlabel NNFrE   )FF)__name__
__module____qualname__strr   boolr_   dictr#   rH   rJ   rZ   ra   rm   ro   ru   r|   __classcell__)rA   s   @rB   r   r      s4    #'  %"(,$"'%#%*.)-&+#P#P# 3-P# 	P#
 P# P# P# P# P#  ~P# P#  P# P# !P# #3-P#  "#!P#"  $#P#d! *.P!#PdB'8&rC   r   )!__doc__r]   os	itertoolsr   r   typingr   rP   torch.distributeddistributedr3   PILr   r&   datasets.distributedr   datasets.splitsr   ImportErroreprintr   r
   readerr   shared_countr   r_   environgetr%   r   rq   rC   rB   <module>r      s     	 #     :) &  % 2::>>"6=>&   	
PQ
Gs   A: :B?
B		B