o 粪g~)@sdZddlZddlmZddlmZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZddlmZmZddlmZdd lmZmZdd lmZdd lmZmZdd lm Z m!Z!dd l"m#Z#edZ$ededZ%de e$de e$ge%fde e$fddZ&GdddeZ'dS)zo Ensemble retriever that ensemble the results of multiple retrievers by using weighted Reciprocal Rank Fusion N) defaultdict)Hashable)chain) AnyCallableDictIterableIteratorListOptionalTypeVarcast)#AsyncCallbackManagerForRetrieverRunCallbackManagerForRetrieverRun)Document) BaseRetriever RetrieverLike)RunnableConfig) ensure_config patch_config)ConfigurableFieldSpecget_unique_config_specs)model_validatorTH)bounditerablekeyreturnccs6t}|D]}||}|vr|||VqdS)a Yield unique elements of an iterable based on a key function. Args: iterable: The iterable to filter. key: A function that returns a hashable key for each element. Yields: Unique elements of the iterable based on the key function. N)setadd)rrseenekr$Y/var/www/html/chatdoc2/venv/lib/python3.10/site-packages/langchain/retrievers/ensemble.py unique_by_key(s  r&c @sreZdZUdZeeed<eeed<dZe ed<dZ e e ed<e deefd d Zed d ed ee efdefddZ d%de de ededeefddZ d%de de ededeefddZde dedeefddZde dedeefddZddde dede edeefddZddde dede edeefd d!Zd"eeedeefd#d$ZdS)&EnsembleRetrieveraeRetriever that ensembles the multiple retrievers. It uses a rank fusion. Args: retrievers: A list of retrievers to ensemble. weights: A list of weights corresponding to the retrievers. Defaults to equal weighting for all retrievers. c: A constant added to the rank, controlling the balance between the importance of high-ranked items and the consideration given to lower-ranked items. Default is 60. id_key: The key in the document's metadata used to determine unique documents. If not specified, page_content is used. retrieversweights<cNid_keyrcCstdd|jDS)z+List configurable fields for this runnable.css |] }|jD]}|VqqdSN) config_specs).0 retrieverspecr$r$r% Qs  z1EnsembleRetriever.config_specs..)rr(selfr$r$r%r.NszEnsembleRetriever.config_specsbefore)modevaluescCs,|dst|d}d|g||d<|S)Nr)r()getlen)clsr7 n_retrieversr$r$r% set_weightsUs  zEnsembleRetriever.set_weightsinputconfigkwargsc Ksddlm}t|}|j|dd|dd|dg|j|di|jd}|jd|fd |d p5|i|}z |j |||d }Wnt yW}z| ||d}~ww|j |fi||S) Nr)CallbackManager callbacksverboseFtagsmetadatarCinheritable_tags local_tagsinheritable_metadatalocal_metadatanamerun_name run_managerr?) langchain_core.callbacksrAr configurer9rDrEon_retriever_startget_name rank_fusion Exceptionon_retriever_erroron_retriever_end) r4r>r?r@rAcallback_managerrNresultr"r$r$r%invoke]s@      zEnsembleRetriever.invokec sddlm}t|}|j|dd|dd|dg|j|di|jd}|jd|fd |d p6|i|IdH}z |j |||d IdH}Wnt ya}z | |IdH|d}~ww|j |fi|IdH|S) Nr)AsyncCallbackManagerrBrCFrDrErFrKrLrM) rOrZrrPr9rDrErQrR arank_fusionrTrUrV) r4r>r?r@rZrWrNrXr"r$r$r%ainvoke~sF       zEnsembleRetriever.ainvokequeryrNcCs|||}|S)z Get the relevant documents for a given query. Args: query: The query to search for. Returns: A list of reranked documents. )rSr4r]rNfused_documentsr$r$r%_get_relevant_documentss z)EnsembleRetriever._get_relevant_documentscs|||IdH}|S)z Asynchronously get the relevant documents for a given query. Args: query: The query to search for. Returns: A list of reranked documents. N)r[r^r$r$r%_aget_relevant_documentssz*EnsembleRetriever._aget_relevant_documents)r?csRfddt|jD}tt|D] }dd||D||<q||}|S)z Retrieve the results of the retrievers and use rank_fusion_func to get the final result. Args: query: The query to search for. Returns: A list of reranked documents. c 6g|]\}}|tjd|dddqS retriever_r8)tag)rB)rYr get_childr/ir0r?r]rNr$r% sz1EnsembleRetriever.rank_fusion..cSs*g|]}t|trttt|dn|qS) page_content) isinstancestrrr r/docr$r$r%rjs) enumerater(ranger:weighted_reciprocal_rankr4r]rNr?retriever_docsrhr_r$rir%rSs  zEnsembleRetriever.rank_fusioncs`tjfddt|jDIdH}tt|D] }dd||D||<q||}|S)z Asynchronously retrieve the results of the retrievers and use rank_fusion_func to get the final result. Args: query: The query to search for. Returns: A list of reranked documents. c rbrc)r\rrfrgrir$r%rjsz2EnsembleRetriever.arank_fusion..NcSs$g|]}t|tst|dn|qSrk)rmrror$r$r%rjs)asynciogatherrqr(rrr:rsrtr$rir%r[s  zEnsembleRetriever.arank_fusion doc_listscst|tjkr tdttt|jD](\}}t|ddD]\}}jdur.|jn|j j||j 7<q!qt |}t t|fdddfddd }|S) a Perform weighted Reciprocal Rank Fusion on multiple rank lists. You can find more details about RRF here: https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf Args: doc_lists: A list of rank lists, where each rank list contains unique items. Returns: list: The final aggregated list of items sorted by their weighted RRF scores in descending order. zFs  z.Tcs"jdur |jS|jjSr-rzr{ rrf_scorer4r$r%r|Ms  )reverser)r:r) ValueErrorrfloatziprqr,rlrEr+r from_iterablesortedr&)r4rxdoc_listweightrankrpall_docs sorted_docsr$r}r%rs s2      z*EnsembleRetriever.weighted_reciprocal_rankr-) __name__ __module__ __qualname____doc__r r__annotations__rr+intr,r rnpropertyrr.r classmethodrrr=rrrYr\rr`rrarSr[rsr$r$r$r%r'9s     " #   . , r')(rrv collectionsrcollections.abcr itertoolsrtypingrrrrr r r r r rOrrlangchain_core.documentsrlangchain_core.retrieversrrlangchain_core.runnablesrlangchain_core.runnables.configrrlangchain_core.runnables.utilsrrpydanticrrrr&r'r$r$r$r%s"   ,    (