o
    Dh+                     @   s   d Z ddlZddlmZ ddlmZmZ ddlmZ e	e
Zdeded	ee d
ee dee f
ddZ				ddee ded	ee d
ee dededee fddZdS )z%
Utilities dedicated to URL sampling
    N)sample)ListOptional   )UrlStoreurlstore
samplesizeexclude_minexclude_maxreturnc              	      s   g }| j D ]Y dd |  D }|r'|durt||k s'|dur1t||kr1td t| qt||kr@tt||d}n|}| fdd|D  td t|t|t|  q|S )z4Iterate through the hosts in store and draw samples.c                 S   s   g | ]}|j d vr| qS ))   /N)urlpathpath.0p r   Q/home/air/segue/gemini/back/venv/lib/python3.10/site-packages/courlan/sampling.py
<listcomp>   s
    
z _make_sample.<locals>.<listcomp>Nzdiscarded (size): %s		urls: %s)kc                    s   g | ]} | qS r   r   r   domainr   r   r   -   s    z%s		urls: %s	prop.: %s)	urldict
_load_urlslenLOGGERwarningsortedr   extenddebug)r   r   r	   r
   output_urlsurlpathsmysampler   r   r   _make_sample   s0   
r#   F
input_urlsstrictverbosec                 C   sF   |r	t tj nt tj tdd||d}||  t||||S )zRSample a list of URLs by domain name, optionally using constraints on their numberTN)
compressedlanguager%   r&   )r   setLevelloggingDEBUGERRORr   add_urlsr#   )r$   r   r	   r
   r%   r&   r   r   r   r   sample_urls7   s   

r.   )NNFF)__doc__r*   randomr   typingr   r   r   r   	getLogger__name__r   intstrr#   boolr.   r   r   r   r   <module>   sH    

*