o ȳgT @sHdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%e rxddl&Z'ddl(m)m*m+Z,e-e.Z/d Z0d Z1d Z2d+ddZ3GdddZ4e4Z5e4Z6Gddde7e Z8Gddde!Z9Gddde!Z:GdddZ;dee7deddd e7fd d!Z?d"edd ee7ddffd#d$Z@eeAe4fZBe7ZCGd%d&d&ZDeDZEe7ZFGd'd(d(e9e:eeEeFfZGee7eefZHeAZIGd)d*d*e9e:eeHeIfZJeGZKeJZLeDZMdS),z*A common module for NVIDIA Riva Runnables.N)Enum) TYPE_CHECKINGAnyAsyncGenerator AsyncIteratorDict GeneratorIteratorListOptionalTupleUnioncast) AnyMessage BaseMessage) PromptValue)RunnableConfigRunnableSerializable) AnyHttpUrl BaseModelField parse_obj_asroot_validator validator?i) .!?¡¿return riva.clientc Cs4zddl}W|jSty}ztd|d}~ww)z5Import the riva client and raise an error on failure.rNziCould not import the NVIDIA Riva client library. Please install it with `pip install nvidia-riva-client`.) riva.client ImportErrorclient)rivaerrr(e/var/www/html/chatdoc2/venv/lib/python3.10/site-packages/langchain_community/utilities/nvidia_riva.py_import_riva_client1s r*c@seZdZdZdS) SentinelTzAn empty Sentinel type.N)__name__ __module__ __qualname____doc__r(r(r(r)r+>sr+c@sLeZdZdZdZdZdZdZdZdZ e de d dfd d Z e dd dZdS)RivaAudioEncodingaAn enum of the possible choices for Riva audio encoding. The list of types exposed by the Riva GRPC Protobuf files can be found with the following commands: ```python import riva.client print(riva.client.AudioEncoding.keys()) # noqa: T201 ``` ALAWENCODING_UNSPECIFIEDFLAC LINEAR_PCMMULAWOGGOPUS format_coder!c CsBz |j|j|jd|WSty }ztd||d}~ww)zReturn the audio encoding specified by the format code in the wave file. ref: https://mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html )z>The following wave file format code is not supported by Riva: N)r4r1r5KeyErrorNotImplementedError)clsr7r'r(r(r)from_wave_format_codeXsz'RivaAudioEncoding.from_wave_format_coderiva.client.AudioEncodingcCst}t|j|S)z-Returns the Riva API object for the encoding.)r*getattr AudioEncodingself riva_clientr(r(r)riva_pb2fs zRivaAudioEncoding.riva_pb2N)r!r?)r,r-r.r/r1r2r3r4r5r6 classmethodintr>propertyrEr(r(r(r)r0Fs  r0c@seZdZUdZeeddddgdZeeefe d<eddd Z e ee d <e dd dZ eddddeded efddZdS) RivaAuthMixinzBConfiguration for the authentication to a Riva service connection.zhttp://localhost:50051z1The full URL where the Riva service can be found.z"https://user@pass:riva.example.com) descriptionexamplesurlNz@A full path to the file where Riva's public ssl key can be read.rJssl_certr!riva.client.AuthcCsBt}tt|j}|jdk}t|jdd}|j|j||dS)z!Return a riva client auth object.https/)rNuse_ssluri) r*rrrLschemestrsplitAuthrN)rCrDrLrS url_no_schemer(r(r)authzs  zRivaAuthMixin.authT)pre allow_reusevalcCs$t|tr tttt|Stt|S)z:Do some initial conversations for the URL before checking.) isinstancerVrrr)r=r]r(r(r) _validate_urls  zRivaAuthMixin._validate_url)r!rO)r,r-r.r/rrrLr rV__annotations__rNr rHrZrrFrr_r(r(r(r)rIms   rIc@sPeZdZUdZeejddZeed<edddZ e ed<edd dZ e ed <d S) RivaCommonConfigMixinz%A collection of common Riva settings.z!The encoding on the audio stream.)defaultrJencodingi@z*The sample rate frequency of audio stream.sample_rate_hertzzen-USzaThe [BCP-47 language code](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) for the target language. language_codeN) r,r-r.r/rr0r4rcr`rdrGrerVr(r(r(r)ras rac@sfeZdZUdZejed<ejed<dddZddd Z dd d Z de fd d Z dddZ dddZdS)_Eventz3A combined event that is threadsafe and async safe._event_aeventr!NcCst|_t|_dS)zInitialize the event.N) threadingEventrgasynciorhrCr(r(r)__init__ z_Event.__init__cC|j|jdSzSet the event.N)rgsetrhrlr(r(r)rqrnz _Event.setcCrorp)rgclearrhrlr(r(r)rrrnz _Event.clearcC |jS)zIndicate if the event is set.)rgis_setrlr(r(r)rts z _Event.is_setcCs|jdS)zWait for the event to be set.N)rgwaitrlr(r(r)rusz _Event.waitcs|jIdHdS)z#Async wait for the event to be set.N)rhrurlr(r(r) async_waitsz_Event.async_waitr!N)r,r-r.r/rirjr`rkrmrqrrboolrtrurvr(r(r(r)rfs       rfoutput_directory sample_ratecCsr|r7tjddd|d }|j}Wdn1swYt|d}|d|d||||fSd S) zECreate a new wave file and return the wave write object and filename.bxz.wavF)modesuffixdeletedirNwbr8rR)NN)tempfileNamedTemporaryFilenamewaveopen setnchannels setsampwidth setframerate)ryrzf wav_file_namewav_filer(r(r) _mk_wave_files    rr] TTSInputTypecCs.t|tr |St|trt|jSt|S)zAttempt to coerce the input value to a string. This is particularly useful for converting LangChain message to strings. )r^r to_stringrrVcontentr]r(r(r)_coerce_strings   rinputsccsd}|D]D}t|}tD]}||vr(||d\}}|||Vd}||vsq ||7}t|tkrItdt|tD] }|||dVq;d}q|rQ|VdSdS)z9Filter the input chunks are return strings ready for TTS.r8rN)r_SENTENCE_TERMINATORSrWlen_MAX_TEXT_LENGTHrange)rbufferchunk terminator last_sentenceidxr(r(r)_process_chunkss(  rc@sZeZdZUdZejed<ejed<ejed<e ed<e ed<e ed<e ej ed<d)d e d d fd dZ d eed d ffddZd eefddZed efddZed efddZed efddZed efddZd*dede e d d fddZd*dede e d d fdd Zd*de e d d fd!d"Zd*de e d d fd#d$Zd%ed&d d fd'd(Zd S)+ AudioStreamz%A message containing streaming audio. _put_lock_queueoutputhangup user_talking user_quiet_workerrmaxsizer!NcCsDt|_tj|d|_t|_t|_t|_ t|_ d|_ dS)zInitialize the queue.)rN) riLockrqueueQueuerrrfrrrr)rCrr(r(r)rm s   zAudioStream.__init__ccsJ z |jdt}Wn tjyYqw|tkrdS|V|jq)zReturn an error.TN)rget_QUEUE_GET_TIMEOUTrEmptyHANGUP task_donerCnext_valr(r(r)__iter__s zAudioStream.__iter__cCsZ ztd|jjdtIdH}Wn tjyYqw|tkr$dS|V|j q)z4Iterate through all items in the queue until HANGUP.TN) rkget_event_looprun_in_executorrrrrrrrrr(r(r) __aiter__&s  zAudioStream.__aiter__cCrs)z(Indicate if the audio stream has hungup.)rrtrlr(r(r)hungup9 zAudioStream.hungupcCrs)z-Indicate in the input stream buffer is empty.)remptyrlr(r(r)r>rzAudioStream.emptycCs4|jo|j}|jduo|j o|j}|o|S)z;Indicate if the audio stream has hungup and been processed.N)rrris_aliver)rC input_done output_doner(r(r)completeCs   zAudioStream.completecCs|jr|jSdS)z&Indicate if the ASR stream is running.F)rrrlr(r(r)runningNs zAudioStream.runningitemtimeoutcCs\|j!|jr td|tur|j|jj||dWddS1s'wYdS)zPut a new item into the queue.z?The audio stream has already been hungup. Cannot put more data.rN)rr RuntimeErrorrrrqrput)rCrrr(r(r)rUs "zAudioStream.putcs,t}t|d|j||IdHdS)z$Async put a new item into the queue.N)rkrwait_forrr)rCrrloopr(r(r)aput`s"zAudioStream.aputcCs|t|dS)zSend the hangup signal.N)rrrCrr(r(r)closeeszAudioStream.closecs|t|IdHdS)zAsync send the hangup signal.N)rrrr(r(r)acloseizAudioStream.aclose responseszrasr.StreamingRecognizeResponsecsZjrtdtjdddd fdd }tj|d _d j_jdS) zIDrain the responses from the provided iterator and put them into a queue.z,An ASR instance has already been registered.rRrrr!NcsD]<}|js q|jD]2}|jsq|jr2jjtt |jdj }j |qj sAjjqqdS)zConsume the ASR Generator.rN)ruresults alternativesis_finalrrrrrqrrV transcriptrrrt)responseresultr has_startedrrCr(r)workerts$      z$AudioStream.register..worker)targetTrw) rrriBarrierThreadrdaemonstartru)rCrrr(rr)registerms  zAudioStream.register)rN) r,r-r.r/rirr`rrrfr rrGrmrbytesrrStreamInputTyperrHrxrrrrrrrrr rr(r(r(r)rs2       rc @seZdZUdZdZeed<dZeed<edddZ e ed <ed d dZ e ed <ed d dZ e ed<ed dedeeefdeeeffddZed ddZd!ddZ d"dedeededefddZdS)#RivaASRzNA runnable that performs Automatic Speech Recognition (ASR) using NVIDIA Riva.nvidia_riva_asrrzA Runnable for converting audio bytes to a string.This is useful for feeding an audio stream into a chain andpreprocessing that audio to create an LLM prompt.rJr8z7The number of audio channels in the input audio stream.rMaudio_channel_countTz\Controls whether or not Riva should attempt to filter profanity out of the transcribed text.profanity_filterz]Controls whether Riva should attempt to correct senetence puncuation in the transcribed text.enable_automatic_punctuationr[valuesr!cC t}|Sz4Validate the Python environment and input arguments.r*r=r_r(r(r)_validate_environmentzRivaASR._validate_environment&riva.client.StreamingRecognitionConfigc Cs4t}|jd|j|j|j|jd|j|j|jddS)z)Create and return the riva config object.Tr8)rcrdrmax_alternativesrrre)interim_resultsconfig) r*StreamingRecognitionConfigRecognitionConfigrcrdrrrrerBr(r(r)rszRivaASR.configriva.client.ASRServicec C8t}z||jWSty}ztd|d}~ww);Connect to the riva service and return the a client object.z5Error raised while connecting to the Riva ASR server.N)r* ASRServicerZ Exception ValueErrorrCrDr'r(r(r) _get_servicezRivaASR._get_serviceNinputrkwargscKs|js|}|j||jd}||g}|jsl|jj|jjd}Wdn1s/wY|ri|j sZz ||j g7}Wn t j yOYq6w|j |j r;tdt|d|S|jrdS)z3Transcribe the audio bytes into a string with Riva.) audio_chunksstreaming_configg?NzRiva ASR returning: %s r)rrstreaming_response_generatorrrrr not_emptyrur get_nowaitrrr_LOGGERdebugreprjoinstrip)rCrrrservicer full_responsereadyr(r(r)invokes2     zRivaASR.invoke)r!r)r!rr)r,r-r.r/rrVr`rJrrrGrrxrrrFrrrrHrr ASRInputTyper r ASROutputTyper r(r(r(r)rs@   &   rc @s0eZdZUdZdZeed<dZeed<edddZ eed <ed d dZ e eed <e d de deeefdeeeffddZed e dedefddZd"ddZ d#dede ededefddZ d#deede ede edeefddZ d#deede ede edeed ffd d!Zd S)$RivaTTSz?A runnable that performs Text-to-Speech (TTS) with NVIDIA Riva.nvidia_riva_ttsrz_A tool for converting text to speech.This is useful for converting LLM output into audio bytes.rJzEnglish-US.Female-1zThe voice model in Riva to use for speech. Pre-trained models are documented in [the Riva documentation](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/tts/tts-overview.html).rM voice_nameNzThe directory where all audio files should be saved. A null value indicates that wave files should not be saved. This is useful for debugging purposes.ryTrrr!cCrrrrr(r(r)r rzRivaTTS._validate_environmentvcCs,|rt|}|jdddt|S|S)NT)parentsexist_ok)pathlibPathmkdirrVabsolute)r=rdirpathr(r(r)_output_directory_validator's   z#RivaTTS._output_directory_validator"riva.client.SpeechSynthesisServicec Cr)rz5Error raised while connecting to the Riva TTS server.N)r*SpeechSynthesisServicerZrrrr(r(r)r0rzRivaTTS._get_servicerrrcKsd|t|gS)zDPerform TTS by taking a string and outputting the entire audio file.)r transformiter)rCrrrr(r(r)r :szRivaTTS.invokec ks|}t|j|j\}}t|D],}td||j||j|j |j j |jd}|D]} t t | j} |r:|| | Vq+q|rM|td|dSdS)zHPerform TTS by taking a stream of characters and streaming output bytes.zRiva TTS chunk: %s)textrrercsample_rate_hzzRiva TTS wrote file: %sN)rrryrdrrrsynthesize_onlinerrercrErraudiowriteframesrawr) rCrrrrrrrrrespr!r(r(r)rCs0    zRivaTTS.transformc  stttd fdd }dttffdd d fdd d fd d }|}|} z td IdH}Wn tj j yVYq=w |t ur`n|Vq>|IdH|IdHdS)zGIntercept async transforms and route them to the synchronous transform.r!Ncs.2z 3dHW}|q6tdS)z#Produce input into the input queue.N) put_nowait_TRANSFORM_ENDr)r input_queuer(r) _producerus  z%RivaTTS.atransform.._producerc3s> zjdd}Wn tjyYqw|tkrdS|Vq)zIterate over the input_queue.TrrN)rrrr%r)r&r(r)_input_iterator{sz+RivaTTS.atransform.._input_iteratorcs*D]}|qtdS)z!Consume the input with transform.N)rr$r%r)r( out_queuerCr(r) _consumers z%RivaTTS.atransform.._consumercsdIdHdS)z"Coroutine that wraps the consumer.N)rr()r*rr(r)_consumer_cororz*RivaTTS.atransform.._consumer_coroTrrw) rkget_running_looprrr r create_taskrr exceptions TimeoutErrorrr%) rCrrrr'r+producerconsumerr]r()r*r(rr&rr)rCr) atransformjs.    zRivaTTS.atransform)r!rr)r,r-r.r/rrVr`rJrrryr rrFrrrrrrrr TTSOutputTyper r rrrr2r(r(r(r)r sd     &     * r )r!r")Nr/rkloggingrrrrirenumrtypingrrrrrrr r r r r rlangchain_core.messagesrrlangchain_core.prompt_valuesrlangchain_core.runnablesrrpydanticrrrrrrr#r&riva.client.proto.riva_asr_pb2r%proto riva_asr_pb2rasr getLoggerr,rrrrr*r+rr%rVr0rIrarffloat Wave_writerrrrrStreamOutputTyperr r rrr3r  NVIDIARivaASR NVIDIARivaTTSNVIDIARivaStreamr(r(r(r)sx 8      '!"    h  $