o ©Ci—'ã@s’ddlZddlZddlZddlZe e¡Zejfdd„Z ej Z dd„Z ejddfdd „Z dd d „Z dd d „Zddd„Z  ddd„Zddd„ZdS)éNcCst d|j|f¡t ¡}|j\}}t |¡}tj|||d}t ||¡} t ¡r8t dt ¡¡t  | ¡} d} |D]3} | jd} |   | ¡|   ||¡\} }|| 7}|  | |¡|   ¡| | 7} t d| t ¡|f¡q<| ¡t dt ¡|| f¡|j|jfS)z¤Computes the exact KNN search results for a dataset that possibly does not fit in RAM but for which we have an iterator that returns it block by block. z%knn_ground_truth queries size %s k=%d©Úkeep_maxúrunning on %d GPUsrú%d db elements, %.3f szGT time: %.3f s (%d vectors))ÚLOGÚinfoÚshapeÚtimeÚfaissÚis_similarity_metricÚ ResultHeapÚ IndexFlatÚ get_num_gpusÚindex_cpu_to_all_gpusÚaddÚsearchÚ add_resultÚresetÚfinalizeÚDÚI)ÚxqÚ db_iteratorÚkÚ metric_typeÚt0ÚnqÚdrÚrhÚindexÚi0ÚxbiÚnirr©r#úw/var/www/html/fyndo/python/python_agents/rag_suite/venv/lib/python3.10/site-packages/faiss/contrib/exhaustive_search.pyÚknn_ground_truths,        r%cCsÚ|j\}}t d|¡t|jdƒ}| ||¡\}}|jtjkr-|dd…|df|k} n |dd…|df|k} |   ¡dkrgt d|   ¡¡t |t j ƒr\|} t  ||j¡}| | ¡| || |¡\} } } t d¡gg}}d}t|ƒD]]}| |s³|jtjkr||dd…f|k  ¡}n ||dd…f|k  ¡}| ||d|…f¡| ||d|…f¡qw| || |d}}| | ||…¡| | ||…¡|d7}qwt  dgdd „|Dƒ¡}|t  |¡t  |¡fS) zÞGPU does not support range search, so we emulate it with knn search + fallback to CPU index. The index_cpu can either be a CPU index or a numpy table that will be used to construct a Flat index if needed. zGPU search %d queriesiNérzCPU search remain %dÚcombinecSóg|]}t|ƒ‘qSr#©Úlen)Ú.0Údir#r#r$Ú ]óz$range_search_gpu..)rrÚdebugÚminÚntotalrrr Ú METRIC_L2ÚsumÚ isinstanceÚnpÚndarrayr rÚ range_searchÚrangeÚappendÚcumsumÚhstack)rÚr2Ú index_gpuÚ index_cpurrrrrÚmaskÚxbÚ lim_remainÚD_remainÚI_remainÚD_resÚI_resÚnrÚiÚnvÚl0Úl1Úlimsr#r#r$Úrange_search_gpu6s<           rLFéÿÿÿÿcs|j\}}t ¡}tj|dd}t ||¡} |dkrt ¡}|r6t d|¡t  ¡} || _ tj | | |d} d} dd„t |ƒDƒ} d d„t |ƒDƒ}|D]r}|jd}|dkrk|   |¡t||| |ƒ\}}}|  ¡n|   |¡|  ||¡\}}}|  ¡|| 7}t |ƒD]'}||||d }}||kr¬| | |||…¡|| |||…¡q…| |7} t d | t ¡|f¡qLtjdd d‰tjddd‰‡fd d„| Dƒ} ‡fdd„|Dƒ}dd„|Dƒ}t|ƒ|ksîJ‚tj|d dd}t |¡|d d…<|t | ¡t |¡fS)z§Computes the range-search search results for a dataset that possibly does not fit in RAM but for which we have an iterator that returns it block by block. Úfloat32©ÚdtyperMr©ÚcoÚngpurcSóg|]}g‘qSr#r#©r+Ú_ir#r#r$r-vóz&range_ground_truth..cSrTr#r#rUr#r#r$r-wrWr&rÚint64có"g|] }|gkr t |¡nˆ‘qSr#©r5r;©r+rG)Úempty_Dr#r$r-Žó"crYr#rZr[)Úempty_Ir#r$r-r]cSr(r#r)r[r#r#r$r-r.Úuint64N)rr r5Úascontiguousarrayr r rrrÚGpuMultipleClonerOptionsÚshardrr8rrLrr7r9Úzerosr*r:r;)rrÚ thresholdrrbrSrrrrrRr=r rrr!r"Úlims_iÚDiÚIiÚjrIrJÚsizesrKr#)r\r^r$Úrange_ground_truthasR       €rjc Csp|r||k}n||k}t |¡}d}t|ƒD]\}} t| ƒ} |||| … ¡||<|| 7}q|||||fS)z select a set of results r)r5Ú zeros_likeÚ enumerateÚintr3) ÚnresÚdisÚidsÚthreshrr?Únew_nresÚorGrFr#r#r$Úthreshold_radius_nres—s   rtc Cs‚|r||k}n||k}t |¡}t|ƒd}t|ƒD]}||||d} } |||| | … ¡||d<q|||||fS)z= restrict range-search results to those below a given radius r&)r5rkr*r8r3) rKrorprqrr?Únew_limsÚnrGrIrJr#r#r$Úthreshold_radius¦s    "rwc Csìt dd„|Dƒ¡}t|ƒ|ksJ‚|r&| t|ƒ|d¡|d|}n | |¡||}|jdkr9t|ƒ}nt|ƒ}t d|¡d}t |ƒD] \}\}}} t ||| ||d\}}} |t|ƒ7}||| f||<qJt d |¡||fS) z‘find radius that reduces number of results to target_nres, and applies it in-place to the result batches used in range_search_max_resultscSsg|]\}}}|‘qSr#r#)r+Ú_ror#r#r$r-¸óz apply_maxres..r&rMrNz setting radius to %srrz. updated previous results, new nb results %d) r5r;r*Ú partitionrPÚfloatrmrr/rlrt) Ú res_batchesÚ target_nresrÚalldisÚradiusÚtotresrGrnrorpr#r#r$Ú apply_maxres´s(     ÿ rcCst|dur|dus J‚td|ƒ}|dur |dusJ‚t|dƒ}|dkr(t ¡}|r@t d|¡t ¡}||_tj|||d} t ¡} d} } d} }}g}|D]}t ¡}|dkrgt ||| |ƒ\}}}n |  ||¡\}}}|dd…|dd…}|t |ƒ7}| t |ƒ7} t ¡}|j t jkr—| d ¡}|t |ƒ7}| |||f¡|durÃ||krÃt d ||f¡t||t |j¡d \}}t ¡}| ||7} | ||7} t d t ¡| | |f¡qRt d | | ||f¡|r||krt||t |j¡d \}}t  dd„|Dƒ¡}t  dd„|Dƒ¡}t  dd„|Dƒ¡}t jt |ƒddd}t  |¡|dd…<||||fS)aPerforms a range search with many queries (given by an iterator) and adjusts the threshold on-the-fly so that the total results table does not grow larger than max_results. If ngpu != 0, the function moves the index to this many GPUs to speed up search. Ngš™™™™™é?gø?rMrrQrr&Úint16z-too many results %d > %d, scaling back radiusrz' [%.3f s] %d queries done, %d resultszBsearch done in %.3f s + %.3f s, total %d results, end threshold %gcSsg|]\}}}|‘qSr#r#©r+Únres_iÚdis_iÚids_ir#r#r$r-ryz,range_search_max_results..cSsg|]\}}}|‘qSr#r#rƒr#r#r$r- rycSsg|]\}}}|‘qSr#r#rƒr#r#r$r-!ryr_rO)rmr rrrrarbrr rLr7r*rPr5rNÚastyper9rr rr/r;rcr:)rÚquery_iteratorrÚ max_resultsÚ min_resultsrbrSÚ clip_to_minrRr=Út_startÚt_searchÚt_post_processÚqtotr€Ú raw_totresr|Úxqirrerfrgr„Út1Út2rnrorprKr#r#r$Úrange_search_max_resultsÐsx          ÿ  þ   ÿÿÿ  þ r”é é Nccs\t|ƒ}|}d}||kr,||||…}|V||kr |d9}|t|ƒ7}||ks dSdS)z¤ produces batches of progressively increasing sizes. This is useful to adjust the search radius progressively without overflowing with intermediate results réNr))rÚstart_bsÚmax_bsrÚbsrGr‘r#r#r$Úexponential_query_iterator)s€ ûr›)F)NNFrF)r•r–)r r Únumpyr5ÚloggingÚ getLoggerÚ__name__rr2r%ÚknnrLrjrtrwrr”r›r#r#r#r$Ús$ "+ ÿ 6   þY