
    4AjT                        d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZmZmZmZmZmZmZ h dZh dZg d	Z G d
 de          ZdS )a  Video/image understanding tool using vision-language models.

Analyzes images or video frames using CLIP, BLIP-2, or LLaVA. Supports
frame description, visual question answering, quality assessment, and
scene classification. Primary use case: visual QA for automated video review.
    )annotationsN)Path)Any)	BaseToolDeterminismExecutionModeResourceProfile
ToolResultToolRuntimeToolStability
ToolStatusToolTier>   .avi.mov.mp4.webm>   .bmp.jpg.png.jpeg.tiff.webp)indooroutdoor	landscape	cityscapeportraitactionzclose-upaerial
underwaternightstudionatureurbanabstractztext-overlayc                     e Zd ZdZdZej        ZdZdZ	e
j        Zej        Zej        Zej        ZddgZdZdgZg d	Zd
dgdddddddg dddddg dddddddidddddddd Zd
dd!dddiddiddid"d#Z ed$d%d&d'd()          Zg d*Zg Zd+Z d,d-gZ!dOd0Z"dPd4Z#dQd6Z$dRd?Z%dSdAZ&dTdDZ'dUdGZ(dVdHZ)dWdJZ*dVdKZ+dXdNZ,d+S )YVideoUnderstandvideo_understandz0.1.0analysistransformerszpython:transformerszpython:torchzFpip install transformers torch  # For CLIP/BLIP-2 visual understandingzvideo-understand)image_description	visual_qaquality_assessmentscene_classificationobject_detectionobject
input_pathstringzPath to image or video file)typedescriptionz:Question to answer about the visual content (for VQA mode))describeqaqualityclassifyr5   z|describe: generate caption; qa: answer query about content; quality: assess technical quality; classify: classify scene type)r3   enumdefaultr4   )clipblip2llavar;   z"Which vision-language model to usearrayr3   integerz`For video input, which frames to analyze. If not provided, samples key frames at even intervals.)r3   itemsr4      z.Maximum number of frames to analyze from video)r3   r:   r4   )r1   querymodemodelframe_indices
max_frames)r3   required
propertieszPer-frame analysis results)framessummaryrC   rD   )r3   rH      i   i   i  F)	cpu_coresram_mbvram_mbdisk_mbnetwork_required)r1   rC   rD   rB   Nz<Compare generated descriptions against actual visual contentz3Verify quality scores match perceived image qualityreturnr   c                d    	 ddl }ddl}t          j        S # t          $ r t          j        cY S w xY w)z4Check if transformers and torch are both importable.r   N)r*   torchr   	AVAILABLEImportErrorUNAVAILABLE)selfr*   rS   s      N/app/working/workspaces/default/OpenMontage/tools/analysis/video_understand.py
get_statuszVideoUnderstand.get_status   sO    	*LLL'' 	* 	* 	*))))	*s    //inputsdict[str, Any]floatc                z    |                     dd          }|                     dd          }|dk    r|dz  S |dz  S )z:Estimate runtime in seconds based on mode and frame count.rF   rA   rC   r5   r7         ?g      @get)rW   rZ   rF   rC   s       rX   estimate_runtimez VideoUnderstand.estimate_runtime   sI    ZZa00
zz&*--9##C    r
   c           
        t          |d                   }|                    dd          }|                    dd          }|                    d          }|                    d          }|                    dd	          }|                                st          d
d|           S |j                                        }|t          v }	|t          v }
|	s3|
s1t          d
d| dt          t          t          z                       S |dk    r|st          d
d          S t          j	                    }	 | 
                    ||	||          }n+# t          $ r}t          d
d|           cY d }~S d }~ww xY w|st          d
d          S 	 |dk    r|                     |          }nl|dk    r|                     ||          }nO|dk    r|                     |||          }n1|dk    r|                     ||          }nt          d
d|           S nT# t           $ r!}t          d
d| d|           cY d }~S d }~wt          $ r}t          d
d|           cY d }~S d }~ww xY wt          j	                    |z
  }|                     ||          }t          d||||dk    r|ndt%          |          dt'          |d          |dk    r|nd           S )Nr1   rC   r5   rD   r;   rB   rE   rF   rA   FzInput file not found: )successerrorzUnsupported file type: z. Supported: r6   z Query is required for 'qa' mode.zFailed to load frames: zNo frames could be extracted.r7   r8   zUnknown mode: zMissing dependency for z: zAnalysis failed: Tmetrics)rI   rJ   rC   rD   frame_countrK   )rd   dataduration_secondsrD   )r   r`   existsr
   suffixlowerVIDEO_EXTENSIONSIMAGE_EXTENSIONSsortedtime_load_frames	Exception_analyze_quality_analyze_describe_analyze_qa_analyze_classifyrU   _build_summarylenround)rW   rZ   r1   rC   
model_namerB   rE   rF   rk   is_videois_imagestartrI   eframe_resultselapsedrJ   s                    rX   executezVideoUnderstand.execute   s   &.//
zz&*--ZZ00


7##

?33ZZa00
  "" 	;z;;   
 "((**---- 	 	Pf P P"()9<L)L"M"MP P    4<<<8   
 		R&&HmZ FF  	R 	R 	Re3PQ3P3PQQQQQQQQQ	R  	Te3RSSSS	Ly   $ 5 5f = =## $ 6 6vz J J $ 0 0U K K## $ 6 6vz J J!%7N7N7NOOOO 	 	 	A
AAaAA          	L 	L 	Le3Jq3J3JKKKKKKKKK	L )++% %%mT::'"'+y'8'8i"=11  #7A.. $	 1 1**t
 
 
 	
sO   7E 
E8E3-E83E8BH 
I)"H>8I)>I)I$I)$I)r   r{   boolrE   list[int] | NonerF   intlistc                    ddl m} |s)|                    |                              d          gS |                     |||          S )z3Load PIL Image objects from an image or video file.r   ImageRGB)PILr   openconvert_extract_video_frames)rW   r1   r{   rE   rF   r   s         rX   rq   zVideoUnderstand._load_frames   sY     	 	;JJz**22599::))*mZPPPrb   
video_pathc                   ddl m} t          j                    5 }t	          |          }|rX|d|         }d                    d |D                       }ddt          |          dd	| d
ddt          |dz            dddg}	nSddt          |          dddddddg
}
ddt          |          dt          |          dd| t          |dz            dddg}	t          j        |	ddd           t          |
                    d                    }g }|d|         D ]=}|                    |                    |                              d                     >|cddd           S # 1 swxY w Y   dS )z.Extract frames from a video file using ffmpeg.r   r   N+c              3  "   K   | ]
}d | dV  dS )zeq(n\,)N ).0idxs     rX   	<genexpr>z8VideoUnderstand._extract_video_frames.<locals>.<genexpr>  s?       ' '),$c$$$' ' ' ' ' 'rb   ffmpegz-iz-vfzselect=''z-vsyncvfrzframe_%04d.pngz-yz	-loglevelre   z-mapz0:v:0z-ccopyz-fnull-z	-frames:vz
thumbnail=T<   )capture_outputtexttimeoutzframe_*.pngr   )r   r   tempfileTemporaryDirectoryr   joinstr
subprocessrunro   globappendr   r   )rW   r   rE   rF   r   tmp_dirtmpframes_to_extractselect_exprcmd	probe_cmdframe_filesimagesfs                 rX   r   z%VideoUnderstand._extract_video_frames   s    	(** (	gw--C $1+:+$>!!hh ' '0A' ' '   dC
OO4k444e..//+w dC
OOGT64	 dC
OOZ4
44..//+w N3t$KKKK !-!8!899KF *- < <ejjmm33E::;;;;Q(	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	 (	s   EE))E-0E-rI   list[dict[str, Any]]c                l   ddl }g }t          |          D ]\  }}|                    ||j                  }|                    |d          }|                    g dg dg dg          }ddlm}	  |	||d	
          }
t          |                    |
                    }t          |                    |                    }t          |	                    |                    }g }|dk     r|
                    d           |dk     r|
                    d           n|dk    r|
                    d           |dk     r|
                    d           |sdnd}|
                    |t          |d          t          |d          t          |d          |||j         d|j         d           |S )z4Assess technical quality using simple image metrics.r   N)dtyperK   )axis)r      r   )r   r   )
convolve2dvalid)rC   d   blurry(   underexposed   overexposed   low_contrastgoodissues_detectedx)frame_index
blur_score
brightnesscontrastr7   issues
resolution)numpy	enumerater>   float64meanscipy.signalr   r\   varstdr   ry   widthheight)rW   rI   npresultsiimgarrgraylaplacian_kernelr   	laplacianr   r   r   quality_issuesquality_labels                   rX   rs   z VideoUnderstand._analyze_quality6  s   '' &	 &	FAs((3bj(11C773Q7''D  "xxJJJ			(JKK//////"
4)9HHHIrvvi0011J rwws||,,J RVVC[[))H  NC%%h///B%%n5555c!!%%m444"}}%%n555*8OFF>OMNN #J22#J22!(A..((!$99SZ99      rb   rz   r   c                   ddl }ddlm}m}m}m}m}m}m}	m	}
 |j
                                        rdnd}|dk    rDd}|                    |          }|                    |                              |          }|||fS |dk    rXd	}|                    |          }|                    ||dk    r|j        n|j        
                              |          }|||fS |dk    rXd}|	                    |          }|
                    ||dk    r|j        n|j        
                              |          }|||fS t!          d|           )z7Load the requested vision-language model and processor.r   N)CLIPProcessor	CLIPModelBlipProcessorBlipForConditionalGenerationBlip2ProcessorBlip2ForConditionalGenerationAutoProcessorAutoModelForCausalLMcudacpur;   zopenai/clip-vit-base-patch32r<   zSalesforce/blip2-opt-2.7b)torch_dtyper=   zllava-hf/llava-1.5-7b-hfzUnknown model: )rS   r*   r   r   r   r   r   r   r   r   r   is_availablefrom_pretrainedtofloat16float32
ValueError)rW   rz   rS   r   r   r   r   r   r   r   r   devicemodel_id	processorrD   s                  rX   _load_modelzVideoUnderstand._load_modeli  s   		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 		
 !:2244?%5H%55h??I--h77::6BBE)V++  2H&66x@@I1AAv7G7GemmU] B  bjj  )V++  1H%55h??I(88v7G7GemmU] 9  bjj  )V++7:77888rb   c                   ddl }|                     |          \  }}}g }t          |          D ]\  }}	|dk    rg d}
 ||
|	dd                              |          }|                                5   |di |}ddd           n# 1 swxY w Y   |j                            d	          d         }|                                                                }|
|         }t          ||                                         d
          }|
                    |||d           |dv r ||	d                              |          }|                                5   |j        di |ddi}ddd           n# 1 swxY w Y   |                    |d          d                                         }|
                    ||d           |S )z!Generate captions for each frame.r   Nr;   )za photo of a personza photo of a landscapeza photo of an objectza photo of textza photo of an animalza photo of a buildingza photo of foodza photo of a vehiclezan abstract imageza dark sceneza bright sceneptTr   r   return_tensorspaddingr   dim   )r   r4   
confidencer<   r=   )r   r   max_new_tokens2   skip_special_tokens)r   r4   r   )rS   r   r   r   no_gradlogits_per_imagesoftmaxargmaxitemry   r   generatebatch_decodestrip)rW   rI   rz   rS   rD   r   r   r   r   r   candidate_textsclip_inputsoutputsprobstop_idxcaptionr   rZ   generated_idss                      rX   rt   z!VideoUnderstand._analyze_describe  s    	#'#3#3J#?#? y&'' $	 $	FAsV### # # (i(TSW  "V**  ]]__ 3 3#e22k22G3 3 3 3 3 3 3 3 3 3 3 3 3 3 3088Q8??B,,..--//)'2"5>#6#6#8#8!<<
#$#*",        111"#dCCCFFvNN]]__ P P$2EN$O$OV$O$OB$O$O$OMP P P P P P P P P P P P P P P#00!t 1  UWW  #$#*      
 s$   5	B

B	B	E<<F 	F 	rB   c                   ddl }|                     |          \  }}}g }t          |          D ]\  }	}
|dk    r|d| g} |||
dd                              |          }|                                5   |di |}ddd           n# 1 swxY w Y   |j                            d	          d         }|d                                         }|                    |	||d
k    rdndt          t          |d|z
            d          d           |dv rd| d} ||
|d                              |          }|                                5   |j        di |ddi}ddd           n# 1 swxY w Y   |                    |d          d                                         }|                    |          r)|t          |          d                                         }|                    |	||d           |S )z#Answer a question about each frame.r   Nr;   znot r   Tr   r   r   r^   yesnor   )r   rB   answerr   r   z
Question: z Answer:r   r   r   r   r   r   )r   rB   r  r   )rS   r   r   r   r  r  r  r  r   ry   maxr  r  r	  
startswithrx   )rW   rI   rz   rB   rS   rD   r   r   r   r   r   
candidatesr  r  r  yes_probpromptrZ   r  r  s                       rX   ru   zVideoUnderstand._analyze_qa  s
    	#'#3#3J#?#? y&'' #	 #	FAsV###^E^^4
'i#Cd  "V**  ]]__ 3 3#e22k22G3 3 3 3 3 3 3 3 3 3 3 3 3 3 3088Q8??B 8==??#$"'/#~~ee4"'Ha(l(C(CQ"G"G	        1115e555"VD  "V**  ]]__ P P$2EN$O$OV$O$OB$O$O$OMP P P P P P P P P P P P P P P"//!t 0  UWW  $$V,, :#CKKLL17799F#$"$       s$   8	BB	B	E22E6	9E6	c           
        ddl }|                     |          \  }}}g }t          |          D ])\  }}	|dk    rd t          D             }
 ||
|	dd                              |          }|                                5   |di |}ddd           n# 1 swxY w Y   |j                            d	          d         }t          t          t          |
                                          d
 d          }|                    ||d         d         t          |d         d         d          d |dd         D             d           !|dv rdd                    t                    z   dz   } ||	|d                              |          }|                                5   |j        di |ddi}ddd           n# 1 swxY w Y   |                    |d          d                                         }|                    |          r)|t%          |          d                                         }|                    ||d           +|S )z*Classify each frame into scene categories.r   Nr;   c                    g | ]}d | d	S )za z scener   )r   cats     rX   
<listcomp>z5VideoUnderstand._analyze_classify.<locals>.<listcomp>  s$    "P"P"P#3#3#3#3"P"P"Prb   r   Tr   r   r   c                    | d         S )Nr   r   )r   s    rX   <lambda>z3VideoUnderstand._analyze_classify.<locals>.<lambda>  s
    !A$ rb   )keyreverser   c                :    g | ]\  }}|t          |d           dS )r   )labelscore)ry   )r   r$  r%  s      rX   r  z5VideoUnderstand._analyze_classify.<locals>.<listcomp>  s<     # # #(E5 #(%q//BB# # #rb   rA   )r   top_categoryr   
categoriesr   z2Classify this image into one of these categories: , z. Category:r  r      r   )r   r&  r   )rS   r   r   SCENE_CATEGORIESr   r  r  r  ro   ziptolistr   ry   r   r  r  r	  r  rx   )rW   rI   rz   rS   rD   r   r   r   r   r   r
  r  r  r  scoredr  rZ   r  categorys                      rX   rv   z!VideoUnderstand._analyze_classify  s]    	#'#3#3J#?#? y&'' ,	 ,	FAsV##"P"P?O"P"P"P'i(TSW  "V**  ]]__ 3 3#e22k22G3 3 3 3 3 3 3 3 3 3 3 3 3 3 3088Q8??B(%,,..99&   
 #$$*1IaL"'q	!a"8"8# #,22A2J# # #	        111Hii 0112#$ 
 #VD  "V**  ]]__ P P$2EN$O$OV$O$OB$O$O$OMP P P P P P P P P P P P P P P$11!t 2  UWW  &&v.. >'F5;;==H#$$,      
 s$   	BB	B	*GG	G	r   rC   c                   t          |          }|dk    r6d |D             }|dk    r|d         S d| dd                    |          z   S |dk    r6d	 |D             }|dk    r|d         S d| d
d                    |          z   S |dk    rpg }|D ]+}|                    |                    dg                      ,|sd| dS t	          t          |                    }d| dd                    |           dS |dk    r9d |D             }	|dk    rd|	d          S d| dd                    |	          z   S d| dS )z6Build a human-readable summary from per-frame results.r5   c                :    g | ]}|                     d d          S )r4    r_   r   rs     rX   r  z2VideoUnderstand._build_summary.<locals>.<listcomp>6  s&    LLLAEE-44LLLrb   r   r   z	Analyzed z frames. Descriptions: z; r6   c                :    g | ]}|                     d d          S )r  r1  r_   r2  s     rX   r  z2VideoUnderstand._build_summary.<locals>.<listcomp><  s&    BBBqquuXr**BBBrb   z frames. Answers: r7   r   zAll z  frame(s) passed quality checks.z frame(s). Issues found: r(  .r8   c                :    g | ]}|                     d d          S )r&  unknownr_   r2  s     rX   r  z2VideoUnderstand._build_summary.<locals>.<listcomp>M  s&    RRRq!%%	::RRRrb   zScene classified as: z frames. Scene categories: z
 frame(s).)rx   r   extendr`   ro   set)
rW   r   rC   ndescriptionsanswers
issues_allr3  unique_issuesr'  s
             rX   rw   zVideoUnderstand._build_summary/  s    :LLmLLLLAvv#A&9q999DIIl<S<SSS4<<BBMBBBGAvvqz!4q444tyy7I7III9J" 7 7!!!%%""5"56666 BAaAAAA"3z??33MSASS		-8P8PSSS :RRMRRRJAvv>z!}>>>:A:::))J''(
 )1((((rb   )rQ   r   )rZ   r[   rQ   r\   )rZ   r[   rQ   r
   )
r1   r   r{   r   rE   r   rF   r   rQ   r   )r   r   rE   r   rF   r   rQ   r   )rI   r   rQ   r   )rz   r   )rI   r   rz   r   rQ   r   )rI   r   rz   r   rB   r   rQ   r   )r   r   rC   r   rQ   r   )-__name__
__module____qualname__nameversionr   ANALYZEtier
capabilityproviderr   EXPERIMENTAL	stabilityr   SYNCexecution_moder   DETERMINISTICdeterminismr   	LOCAL_GPUruntimedependenciesinstall_instructionsagent_skillscapabilitiesinput_schemaoutput_schemar	   resource_profileidempotency_key_fieldsside_effectsfallbackuser_visible_verificationrY   ra   r   rq   r   rs   r   rt   ru   rv   rw   r   rb   rX   r'   r'   &   s       DGDJH*I"'N+K#G)>:LP  ''L  L !N !< 
 ![ 
 !AAA%W  !222!C	    ),M	  "O A%
 %
) )LX   ;  )X&h'
 
 M '   FEELHF=!
* * * *       R
 R
 R
 R
pQ Q Q Q1 1 1 1n- - - -f&9 &9 &9 &9P/ / / /b. . . .`7 7 7 7z&) &) &) &) &) &)rb   r'   )__doc__
__future__r   r   r   rp   pathlibr   typingr   tools.base_toolr   r   r   r	   r
   r   r   r   r   rm   rn   r*  r'   r   rb   rX   <module>r`     s=    # " " " " "                  
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 544 FFF    o) o) o) o) o)h o) o) o) o) o)rb   