vision_agent.tools

vision_agent.tools

register_tool

register_tool(imports=None)

Source code in vision_agent/tools/__init__.py

def register_tool(imports: Optional[List] = None) -> Callable:
    def decorator(tool: Callable) -> Callable:
        import inspect

        from .tools import (  # noqa: F811
            get_tool_descriptions,
            get_tools_df,
            get_tools_info,
        )

        global TOOLS, TOOLS_DF, TOOL_DESCRIPTIONS, TOOL_DOCSTRING, TOOLS_INFO

        if tool not in TOOLS:
            TOOLS.append(tool)
            TOOLS_DF = get_tools_df(TOOLS)  # type: ignore
            TOOL_DESCRIPTIONS = get_tool_descriptions(TOOLS)  # type: ignore
            TOOL_DOCSTRING = get_tool_documentation(TOOLS)  # type: ignore
            TOOLS_INFO = get_tools_info(TOOLS)  # type: ignore

            globals()[tool.__name__] = tool
            if imports is not None:
                for import_ in imports:
                    __new_tools__.append(import_)
            __new_tools__.append(inspect.getsource(tool))
        return tool

    return decorator

vision_agent.tools.tools

COLORS module-attribute

COLORS = [
    (158, 218, 229),
    (219, 219, 141),
    (23, 190, 207),
    (188, 189, 34),
    (199, 199, 199),
    (247, 182, 210),
    (127, 127, 127),
    (227, 119, 194),
    (196, 156, 148),
    (197, 176, 213),
    (140, 86, 75),
    (148, 103, 189),
    (255, 152, 150),
    (152, 223, 138),
    (214, 39, 40),
    (44, 160, 44),
    (255, 187, 120),
    (174, 199, 232),
    (255, 127, 14),
    (31, 119, 180),
]

FUNCTION_TOOLS module-attribute

FUNCTION_TOOLS = [
    owl_v2_image,
    owl_v2_video,
    ocr,
    vit_image_classification,
    vit_nsfw_classification,
    countgd_counting,
    florence2_ocr,
    florence2_sam2_image,
    florence2_sam2_video_tracking,
    florence2_phrase_grounding,
    claude35_text_extraction,
    detr_segmentation,
    depth_anything_v2,
    generate_pose_image,
    closest_mask_distance,
    closest_box_distance,
    qwen2_vl_images_vqa,
    qwen2_vl_video_vqa,
    video_temporal_localization,
    flux_image_inpainting,
    siglip_classification,
]

UTIL_TOOLS module-attribute

UTIL_TOOLS = [
    extract_frames_and_timestamps,
    save_json,
    load_image,
    save_image,
    save_video,
    overlay_bounding_boxes,
    overlay_segmentation_masks,
    overlay_heat_map,
]

TOOLS module-attribute

TOOLS = FUNCTION_TOOLS + UTIL_TOOLS

TOOLS_DF module-attribute

TOOLS_DF = get_tools_df(TOOLS)

TOOL_DESCRIPTIONS module-attribute

TOOL_DESCRIPTIONS = get_tool_descriptions(TOOLS)

TOOL_DOCSTRING module-attribute

TOOL_DOCSTRING = get_tool_documentation(TOOLS)

TOOLS_INFO module-attribute

TOOLS_INFO = get_tools_info(FUNCTION_TOOLS)

UTILITIES_DOCSTRING module-attribute

UTILITIES_DOCSTRING = get_tool_documentation(UTIL_TOOLS)

grounding_dino

grounding_dino(
    prompt,
    image,
    box_threshold=0.2,
    iou_threshold=0.2,
    model_size="large",
)

'grounding_dino' is a tool that can detect and count multiple objects given a text prompt such as category names or referring expressions. The categories in text prompt are separated by commas or periods. It returns a list of bounding boxes with normalized coordinates, label names and associated probability scores.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the image. TYPE: str
image	The image to ground the prompt to. TYPE: ndarray
box_threshold	The threshold for the box detection. Defaults to 0.20. TYPE: float DEFAULT: 0.2
iou_threshold	The threshold for the Intersection over Union (IoU). Defaults to 0.20. TYPE: float DEFAULT: 0.2
model_size	The size of the model to use. TYPE: str DEFAULT: 'large'

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> grounding_dino("car. dinosaur", image)
[
    {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
    {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
]

Source code in vision_agent/tools/tools.py

def grounding_dino(
    prompt: str,
    image: np.ndarray,
    box_threshold: float = 0.20,
    iou_threshold: float = 0.20,
    model_size: str = "large",
) -> List[Dict[str, Any]]:
    """'grounding_dino' is a tool that can detect and count multiple objects given a text
    prompt such as category names or referring expressions. The categories in text prompt
    are separated by commas or periods. It returns a list of bounding boxes with
    normalized coordinates, label names and associated probability scores.

    Parameters:
        prompt (str): The prompt to ground to the image.
        image (np.ndarray): The image to ground the prompt to.
        box_threshold (float, optional): The threshold for the box detection. Defaults
            to 0.20.
        iou_threshold (float, optional): The threshold for the Intersection over Union
            (IoU). Defaults to 0.20.
        model_size (str, optional): The size of the model to use.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label, and
            bounding box of the detected objects with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box.

    Example
    -------
        >>> grounding_dino("car. dinosaur", image)
        [
            {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
            {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
        ]
    """
    image_size = image.shape[:2]
    image_b64 = convert_to_b64(image)
    if model_size not in ["large", "tiny"]:
        raise ValueError("model_size must be either 'large' or 'tiny'")
    request_data = {
        "prompt": prompt,
        "image": image_b64,
        "tool": (
            "visual_grounding" if model_size == "large" else "visual_grounding_tiny"
        ),
        "kwargs": {"box_threshold": box_threshold, "iou_threshold": iou_threshold},
        "function_name": "grounding_dino",
    }
    data: Dict[str, Any] = send_inference_request(request_data, "tools")
    return_data = []
    for i in range(len(data["bboxes"])):
        return_data.append(
            {
                "score": round(data["scores"][i], 2),
                "label": data["labels"][i],
                "bbox": normalize_bbox(data["bboxes"][i], image_size),
            }
        )
    return return_data

owl_v2_image

owl_v2_image(
    prompt, image, box_threshold=0.1, fine_tune_id=None
)

'owl_v2_image' is a tool that can detect and count multiple objects given a text prompt such as category names or referring expressions on images. The categories in text prompt are separated by commas. It returns a list of bounding boxes with normalized coordinates, label names and associated probability scores.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the image. TYPE: str
image	The image to ground the prompt to. TYPE: ndarray
box_threshold	The threshold for the box detection. Defaults to 0.10. TYPE: float DEFAULT: 0.1
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> owl_v2_image("car, dinosaur", image)
[
    {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
    {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
]

Source code in vision_agent/tools/tools.py

def owl_v2_image(
    prompt: str,
    image: np.ndarray,
    box_threshold: float = 0.10,
    fine_tune_id: Optional[str] = None,
) -> List[Dict[str, Any]]:
    """'owl_v2_image' is a tool that can detect and count multiple objects given a text
    prompt such as category names or referring expressions on images. The categories in
    text prompt are separated by commas. It returns a list of bounding boxes with
    normalized coordinates, label names and associated probability scores.

    Parameters:
        prompt (str): The prompt to ground to the image.
        image (np.ndarray): The image to ground the prompt to.
        box_threshold (float, optional): The threshold for the box detection. Defaults
            to 0.10.
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label, and
            bounding box of the detected objects with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box.

    Example
    -------
        >>> owl_v2_image("car, dinosaur", image)
        [
            {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
            {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
        ]
    """

    image_size = image.shape[:2]
    if image_size[0] < 1 or image_size[1] < 1:
        return []

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    payload = {
        "prompts": [s.strip() for s in prompt.split(",")],
        "confidence": box_threshold,
        "model": "owlv2",
    }
    metadata = {"function_name": "owl_v2_image"}

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        # we can only execute fine-tuned models with florence2
        payload = {
            "prompts": payload["prompts"],
            "jobId": fine_tune_id,
            "model": "florence2",
        }

    detections = send_task_inference_request(
        payload,
        "text-to-object-detection",
        files=files,
        metadata=metadata,
    )

    # get the first frame
    bboxes = detections[0]
    bboxes_formatted = [
        ODResponseData(
            label=bbox["label"],
            bbox=normalize_bbox(bbox["bounding_box"], image_size),
            score=round(bbox["score"], 2),
        )
        for bbox in bboxes
    ]
    return [bbox.model_dump() for bbox in bboxes_formatted]

owl_v2_video

owl_v2_video(
    prompt, frames, box_threshold=0.1, fine_tune_id=None
)

'owl_v2_video' will run owl_v2 on each frame of a video. It can detect multiple objects independently per frame given a text prompt such as a category name or referring expression but does not track objects across frames. The categories in text prompt are separated by commas. It returns a list of lists where each inner list contains the score, label, and bounding box of the detections for that frame.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the video. TYPE: str
frames	The list of frames to ground the prompt to. TYPE: List[ndarray]
box_threshold	The threshold for the box detection. Defaults to 0.30. TYPE: float DEFAULT: 0.1
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[List[Dict[str, Any]]]	List[List[Dict[str, Any]]]: A list of lists of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> owl_v2_video("car, dinosaur", frames)
[
    [
        {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
        {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
    ],
    ...
]

Source code in vision_agent/tools/tools.py

def owl_v2_video(
    prompt: str,
    frames: List[np.ndarray],
    box_threshold: float = 0.10,
    fine_tune_id: Optional[str] = None,
) -> List[List[Dict[str, Any]]]:
    """'owl_v2_video' will run owl_v2 on each frame of a video. It can detect multiple
    objects independently per frame given a text prompt such as a category name or
    referring expression but does not track objects across frames. The categories in
    text prompt are separated by commas. It returns a list of lists where each inner
    list contains the score, label, and bounding box of the detections for that frame.

    Parameters:
        prompt (str): The prompt to ground to the video.
        frames (List[np.ndarray]): The list of frames to ground the prompt to.
        box_threshold (float, optional): The threshold for the box detection. Defaults
            to 0.30.
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[List[Dict[str, Any]]]: A list of lists of dictionaries containing the
            score, label, and bounding box of the detected objects with normalized
            coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the
            coordinates of the top-left and xmax and ymax are the coordinates of the
            bottom-right of the bounding box.

    Example
    -------
        >>> owl_v2_video("car, dinosaur", frames)
        [
            [
                {'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]},
                {'score': 0.98, 'label': 'car', 'bbox': [0.2, 0.21, 0.45, 0.5},
            ],
            ...
        ]
    """
    if len(frames) == 0 or not isinstance(frames, List):
        raise ValueError("Must provide a list of numpy arrays for frames")

    image_size = frames[0].shape[:2]
    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload = {
        "prompts": [s.strip() for s in prompt.split(",")],
        "confidence": box_threshold,
        "model": "owlv2",
    }
    metadata = {"function_name": "owl_v2_video"}

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        # we can only execute fine-tuned models with florence2
        payload = {
            "prompts": payload["prompts"],
            "jobId": fine_tune_id,
            "model": "florence2",
        }

    detections = send_task_inference_request(
        payload,
        "text-to-object-detection",
        files=files,
        metadata=metadata,
    )

    bboxes_formatted = []
    for frame_data in detections:
        bboxes_formatted_per_frame = [
            ODResponseData(
                label=bbox["label"],
                bbox=normalize_bbox(bbox["bounding_box"], image_size),
                score=round(bbox["score"], 2),
            )
            for bbox in frame_data
        ]
        bboxes_formatted.append(bboxes_formatted_per_frame)
    return [[bbox.model_dump() for bbox in frame] for frame in bboxes_formatted]

grounding_sam

grounding_sam(
    prompt, image, box_threshold=0.2, iou_threshold=0.2
)

'grounding_sam' is a tool that can segment multiple objects given a text prompt such as category names or referring expressions. The categories in text prompt are separated by commas or periods. It returns a list of bounding boxes, label names, mask file names and associated probability scores.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the image. TYPE: str
image	The image to ground the prompt to. TYPE: ndarray
box_threshold	The threshold for the box detection. Defaults to 0.20. TYPE: float DEFAULT: 0.2
iou_threshold	The threshold for the Intersection over Union (IoU). Defaults to 0.20. TYPE: float DEFAULT: 0.2

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, bounding box, and mask of the detected objects with normalized coordinates (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box. The mask is binary 2D numpy array where 1 indicates the object and 0 indicates the background.

Example

>>> grounding_sam("car. dinosaur", image)
[
    {
        'score': 0.99,
        'label': 'dinosaur',
        'bbox': [0.1, 0.11, 0.35, 0.4],
        'mask': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
    },
]

Source code in vision_agent/tools/tools.py

def grounding_sam(
    prompt: str,
    image: np.ndarray,
    box_threshold: float = 0.20,
    iou_threshold: float = 0.20,
) -> List[Dict[str, Any]]:
    """'grounding_sam' is a tool that can segment multiple objects given a text prompt
    such as category names or referring expressions. The categories in text prompt are
    separated by commas or periods. It returns a list of bounding boxes, label names,
    mask file names and associated probability scores.

    Parameters:
        prompt (str): The prompt to ground to the image.
        image (np.ndarray): The image to ground the prompt to.
        box_threshold (float, optional): The threshold for the box detection. Defaults
            to 0.20.
        iou_threshold (float, optional): The threshold for the Intersection over Union
            (IoU). Defaults to 0.20.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label,
            bounding box, and mask of the detected objects with normalized coordinates
            (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left
            and xmax and ymax are the coordinates of the bottom-right of the bounding box.
            The mask is binary 2D numpy array where 1 indicates the object and 0 indicates
            the background.

    Example
    -------
        >>> grounding_sam("car. dinosaur", image)
        [
            {
                'score': 0.99,
                'label': 'dinosaur',
                'bbox': [0.1, 0.11, 0.35, 0.4],
                'mask': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
            },
        ]
    """
    image_size = image.shape[:2]
    image_b64 = convert_to_b64(image)
    request_data = {
        "prompt": prompt,
        "image": image_b64,
        "tool": "visual_grounding_segment",
        "kwargs": {"box_threshold": box_threshold, "iou_threshold": iou_threshold},
        "function_name": "grounding_sam",
    }
    data: Dict[str, Any] = send_inference_request(request_data, "tools")
    return_data = []
    for i in range(len(data["bboxes"])):
        return_data.append(
            {
                "score": round(data["scores"][i], 2),
                "label": data["labels"][i],
                "bbox": normalize_bbox(data["bboxes"][i], image_size),
                "mask": rle_decode(mask_rle=data["masks"][i], shape=data["mask_shape"]),
            }
        )
    return return_data

florence2_sam2_image

florence2_sam2_image(prompt, image, fine_tune_id=None)

'florence2_sam2_image' is a tool that can segment multiple objects given a text prompt such as category names or referring expressions. The categories in the text prompt are separated by commas. It returns a list of bounding boxes, label names, mask file names and associated probability scores of 1.0.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the image. TYPE: str
image	The image to ground the prompt to. TYPE: ndarray
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, bounding box, and mask of the detected objects with normalized coordinates (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box. The mask is binary 2D numpy array where 1 indicates the object and 0 indicates the background.

Example

>>> florence2_sam2_image("car, dinosaur", image)
[
    {
        'score': 1.0,
        'label': 'dinosaur',
        'bbox': [0.1, 0.11, 0.35, 0.4],
        'mask': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
    },
]

Source code in vision_agent/tools/tools.py

def florence2_sam2_image(
    prompt: str, image: np.ndarray, fine_tune_id: Optional[str] = None
) -> List[Dict[str, Any]]:
    """'florence2_sam2_image' is a tool that can segment multiple objects given a text
    prompt such as category names or referring expressions. The categories in the text
    prompt are separated by commas. It returns a list of bounding boxes, label names,
    mask file names and associated probability scores of 1.0.

    Parameters:
        prompt (str): The prompt to ground to the image.
        image (np.ndarray): The image to ground the prompt to.
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label,
            bounding box, and mask of the detected objects with normalized coordinates
            (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left
            and xmax and ymax are the coordinates of the bottom-right of the bounding box.
            The mask is binary 2D numpy array where 1 indicates the object and 0 indicates
            the background.

    Example
    -------
        >>> florence2_sam2_image("car, dinosaur", image)
        [
            {
                'score': 1.0,
                'label': 'dinosaur',
                'bbox': [0.1, 0.11, 0.35, 0.4],
                'mask': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
            },
        ]
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        return []

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    payload = {
        "prompt": prompt,
        "model": "florence2sam2",
    }
    metadata = {"function_name": "florence2_sam2_image"}

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        payload["jobId"] = fine_tune_id

    detections = send_task_inference_request(
        payload,
        "text-to-instance-segmentation",
        files=files,
        metadata=metadata,
    )

    # get the first frame
    frame = detections[0]
    return_data = []
    for detection in frame:
        mask = rle_decode_array(detection["mask"])
        label = detection["label"]
        bbox = normalize_bbox(detection["bounding_box"], detection["mask"]["size"])
        return_data.append({"label": label, "bbox": bbox, "mask": mask, "score": 1.0})
    return return_data

florence2_sam2_video_tracking

florence2_sam2_video_tracking(
    prompt, frames, chunk_length=10, fine_tune_id=None
)

'florence2_sam2_video_tracking' is a tool that can segment and track multiple entities in a video given a text prompt such as category names or referring expressions. You can optionally separate the categories in the text with commas. It can find new objects every 'chunk_length' frames and is useful for tracking and counting without duplicating counts and always outputs scores of 1.0.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the video. TYPE: str
frames	The list of frames to ground the prompt to. TYPE: List[ndarray]
chunk_length	The number of frames to re-run florence2 to find new objects. TYPE: Optional[int] DEFAULT: 10
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[List[Dict[str, Any]]]	List[List[Dict[str, Any]]]: A list of list of dictionaries containing the
List[List[Dict[str, Any]]]	label,segment mask and bounding boxes. The outer list represents each frame and
List[List[Dict[str, Any]]]	the inner list is the entities per frame. The label contains the object ID
List[List[Dict[str, Any]]]	followed by the label name. The objects are only identified in the first framed
List[List[Dict[str, Any]]]	and tracked throughout the video.

Example

>>> florence2_sam2_video("car, dinosaur", frames)
[
    [
        {
            'label': '0: dinosaur',
            'bbox': [0.1, 0.11, 0.35, 0.4],
            'mask': array([[0, 0, 0, ..., 0, 0, 0],
                [0, 0, 0, ..., 0, 0, 0],
                ...,
                [0, 0, 0, ..., 0, 0, 0],
                [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
        },
    ],
    ...
]

Source code in vision_agent/tools/tools.py

def florence2_sam2_video_tracking(
    prompt: str,
    frames: List[np.ndarray],
    chunk_length: Optional[int] = 10,
    fine_tune_id: Optional[str] = None,
) -> List[List[Dict[str, Any]]]:
    """'florence2_sam2_video_tracking' is a tool that can segment and track multiple
    entities in a video given a text prompt such as category names or referring
    expressions. You can optionally separate the categories in the text with commas. It
    can find new objects every 'chunk_length' frames and is useful for tracking and
    counting without duplicating counts and always outputs scores of 1.0.

    Parameters:
        prompt (str): The prompt to ground to the video.
        frames (List[np.ndarray]): The list of frames to ground the prompt to.
        chunk_length (Optional[int]): The number of frames to re-run florence2 to find
            new objects.
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[List[Dict[str, Any]]]: A list of list of dictionaries containing the
        label,segment mask and bounding boxes. The outer list represents each frame and
        the inner list is the entities per frame. The label contains the object ID
        followed by the label name. The objects are only identified in the first framed
        and tracked throughout the video.

    Example
    -------
        >>> florence2_sam2_video("car, dinosaur", frames)
        [
            [
                {
                    'label': '0: dinosaur',
                    'bbox': [0.1, 0.11, 0.35, 0.4],
                    'mask': array([[0, 0, 0, ..., 0, 0, 0],
                        [0, 0, 0, ..., 0, 0, 0],
                        ...,
                        [0, 0, 0, ..., 0, 0, 0],
                        [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
                },
            ],
            ...
        ]
    """
    if len(frames) == 0 or not isinstance(frames, List):
        raise ValueError("Must provide a list of numpy arrays for frames")

    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload = {
        "prompt": prompt,
        "model": "florence2sam2",
    }
    metadata = {"function_name": "florence2_sam2_video_tracking"}

    if chunk_length is not None:
        payload["chunk_length_frames"] = chunk_length  # type: ignore

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        payload["jobId"] = fine_tune_id

    detections = send_task_inference_request(
        payload,
        "text-to-instance-segmentation",
        files=files,
        metadata=metadata,
    )

    return_data = []
    for frame in detections:
        return_frame_data = []
        for detection in frame:
            mask = rle_decode_array(detection["mask"])
            label = str(detection["id"]) + ": " + detection["label"]
            return_frame_data.append({"label": label, "mask": mask, "score": 1.0})
        return_data.append(return_frame_data)
    return_data = add_bboxes_from_masks(return_data)
    return nms(return_data, iou_threshold=0.95)

ocr

ocr(image)

'ocr' extracts text from an image. It returns a list of detected text, bounding boxes with normalized coordinates, and confidence scores. The results are sorted from top-left to bottom right.

PARAMETER	DESCRIPTION
image	The image to extract text from. TYPE: ndarray

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the detected text, bbox with normalized coordinates, and confidence score.

Example

>>> ocr(image)
[
    {'label': 'hello world', 'bbox': [0.1, 0.11, 0.35, 0.4], 'score': 0.99},
]

Source code in vision_agent/tools/tools.py

def ocr(image: np.ndarray) -> List[Dict[str, Any]]:
    """'ocr' extracts text from an image. It returns a list of detected text, bounding
    boxes with normalized coordinates, and confidence scores. The results are sorted
    from top-left to bottom right.

    Parameters:
        image (np.ndarray): The image to extract text from.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the detected text, bbox
            with normalized coordinates, and confidence score.

    Example
    -------
        >>> ocr(image)
        [
            {'label': 'hello world', 'bbox': [0.1, 0.11, 0.35, 0.4], 'score': 0.99},
        ]
    """

    pil_image = Image.fromarray(image).convert("RGB")
    image_size = pil_image.size[::-1]
    if image_size[0] < 1 or image_size[1] < 1:
        return []
    image_buffer = io.BytesIO()
    pil_image.save(image_buffer, format="PNG")
    buffer_bytes = image_buffer.getvalue()
    image_buffer.close()

    res = requests.post(
        _OCR_URL,
        files={"images": buffer_bytes},
        data={"language": "en"},
        headers={"contentType": "multipart/form-data", "apikey": _API_KEY},
    )

    if res.status_code != 200:
        raise ValueError(f"OCR request failed with status code {res.status_code}")

    data = res.json()
    output = []
    for det in data[0]:
        label = det["text"]
        box = [
            det["location"][0]["x"],
            det["location"][0]["y"],
            det["location"][2]["x"],
            det["location"][2]["y"],
        ]
        box = normalize_bbox(box, image_size)
        output.append({"label": label, "bbox": box, "score": round(det["score"], 2)})

    ocr_results = sorted(output, key=lambda x: (x["bbox"][1], x["bbox"][0]))
    return ocr_results

loca_zero_shot_counting

loca_zero_shot_counting(image)

'loca_zero_shot_counting' is a tool that counts the dominant foreground object given an image and no other information about the content. It returns only the count of the objects in the image.

PARAMETER	DESCRIPTION
image	The image that contains lot of instances of a single object TYPE: ndarray

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the key 'count' and the count as a value, e.g. {count: 12} and a heat map for visualization purposes.

Example

>>> loca_zero_shot_counting(image)
{'count': 83,
'heat_map': array([[ 0,  0,  0, ...,  0,  0,  0],
    [ 0,  0,  0, ...,  0,  0,  0],
    [ 0,  0,  0, ...,  0,  0,  1],
    ...,
    [ 0,  0,  0, ..., 30, 35, 41],
    [ 0,  0,  0, ..., 41, 47, 53],
    [ 0,  0,  0, ..., 53, 59, 64]], dtype=uint8)}

Source code in vision_agent/tools/tools.py

def loca_zero_shot_counting(image: np.ndarray) -> Dict[str, Any]:
    """'loca_zero_shot_counting' is a tool that counts the dominant foreground object given
    an image and no other information about the content. It returns only the count of
    the objects in the image.

    Parameters:
        image (np.ndarray): The image that contains lot of instances of a single object

    Returns:
        Dict[str, Any]: A dictionary containing the key 'count' and the count as a
            value, e.g. {count: 12} and a heat map for visualization purposes.

    Example
    -------
        >>> loca_zero_shot_counting(image)
        {'count': 83,
        'heat_map': array([[ 0,  0,  0, ...,  0,  0,  0],
            [ 0,  0,  0, ...,  0,  0,  0],
            [ 0,  0,  0, ...,  0,  0,  1],
            ...,
            [ 0,  0,  0, ..., 30, 35, 41],
            [ 0,  0,  0, ..., 41, 47, 53],
            [ 0,  0,  0, ..., 53, 59, 64]], dtype=uint8)}
    """

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "function_name": "loca_zero_shot_counting",
    }
    resp_data: dict[str, Any] = send_inference_request(data, "loca", v2=True)
    resp_data["heat_map"] = np.array(resp_data["heat_map"][0]).astype(np.uint8)
    return resp_data

loca_visual_prompt_counting

loca_visual_prompt_counting(image, visual_prompt)

'loca_visual_prompt_counting' is a tool that counts the dominant foreground object given an image and a visual prompt which is a bounding box describing the object. It returns only the count of the objects in the image.

PARAMETER	DESCRIPTION
image	The image that contains lot of instances of a single object visual_prompt (Dict[str, List[float]]): Bounding box of the object in format [xmin, ymin, xmax, ymax]. Only 1 bounding box can be provided. TYPE: ndarray

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the key 'count' and the count as a value, e.g. {count: 12} and a heat map for visualization purposes.

Example

>>> loca_visual_prompt_counting(image, {"bbox": [0.1, 0.1, 0.4, 0.42]})
{'count': 83,
'heat_map': array([[ 0,  0,  0, ...,  0,  0,  0],
    [ 0,  0,  0, ...,  0,  0,  0],
    [ 0,  0,  0, ...,  0,  0,  1],
    ...,
    [ 0,  0,  0, ..., 30, 35, 41],
    [ 0,  0,  0, ..., 41, 47, 53],
    [ 0,  0,  0, ..., 53, 59, 64]], dtype=uint8)}

Source code in vision_agent/tools/tools.py

def loca_visual_prompt_counting(
    image: np.ndarray, visual_prompt: Dict[str, List[float]]
) -> Dict[str, Any]:
    """'loca_visual_prompt_counting' is a tool that counts the dominant foreground object
    given an image and a visual prompt which is a bounding box describing the object.
    It returns only the count of the objects in the image.

    Parameters:
        image (np.ndarray): The image that contains lot of instances of a single object
            visual_prompt (Dict[str, List[float]]): Bounding box of the object in
            format [xmin, ymin, xmax, ymax]. Only 1 bounding box can be provided.

    Returns:
        Dict[str, Any]: A dictionary containing the key 'count' and the count as a
            value, e.g. {count: 12} and a heat map for visualization purposes.

    Example
    -------
        >>> loca_visual_prompt_counting(image, {"bbox": [0.1, 0.1, 0.4, 0.42]})
        {'count': 83,
        'heat_map': array([[ 0,  0,  0, ...,  0,  0,  0],
            [ 0,  0,  0, ...,  0,  0,  0],
            [ 0,  0,  0, ...,  0,  0,  1],
            ...,
            [ 0,  0,  0, ..., 30, 35, 41],
            [ 0,  0,  0, ..., 41, 47, 53],
            [ 0,  0,  0, ..., 53, 59, 64]], dtype=uint8)}
    """

    image_size = get_image_size(image)
    bbox = visual_prompt["bbox"]
    image_b64 = convert_to_b64(image)

    data = {
        "image": image_b64,
        "bbox": list(map(int, denormalize_bbox(bbox, image_size))),
        "function_name": "loca_visual_prompt_counting",
    }
    resp_data: dict[str, Any] = send_inference_request(data, "loca", v2=True)
    resp_data["heat_map"] = np.array(resp_data["heat_map"][0]).astype(np.uint8)
    return resp_data

countgd_counting

countgd_counting(prompt, image, box_threshold=0.23)

'countgd_counting' is a tool that can detect multiple instances of an object given a text prompt. It is particularly useful when trying to detect and count a large number of objects. It returns a list of bounding boxes with normalized coordinates, label names and associated confidence scores.

PARAMETER	DESCRIPTION
prompt	The object that needs to be counted. TYPE: str
image	The image that contains multiple instances of the object. TYPE: ndarray
box_threshold	The threshold for detection. Defaults to 0.23. TYPE: float DEFAULT: 0.23

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> countgd_counting("flower", image)
[
    {'score': 0.49, 'label': 'flower', 'bbox': [0.1, 0.11, 0.35, 0.4]},
    {'score': 0.68, 'label': 'flower', 'bbox': [0.2, 0.21, 0.45, 0.5},
    {'score': 0.78, 'label': 'flower', 'bbox': [0.3, 0.35, 0.48, 0.52},
    {'score': 0.98, 'label': 'flower', 'bbox': [0.44, 0.24, 0.49, 0.58},
]

Source code in vision_agent/tools/tools.py

def countgd_counting(
    prompt: str,
    image: np.ndarray,
    box_threshold: float = 0.23,
) -> List[Dict[str, Any]]:
    """'countgd_counting' is a tool that can detect multiple instances of an object
    given a text prompt. It is particularly useful when trying to detect and count a
    large number of objects. It returns a list of bounding boxes with normalized
    coordinates, label names and associated confidence scores.

    Parameters:
        prompt (str): The object that needs to be counted.
        image (np.ndarray): The image that contains multiple instances of the object.
        box_threshold (float, optional): The threshold for detection. Defaults
            to 0.23.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label, and
            bounding box of the detected objects with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box.

    Example
    -------
        >>> countgd_counting("flower", image)
        [
            {'score': 0.49, 'label': 'flower', 'bbox': [0.1, 0.11, 0.35, 0.4]},
            {'score': 0.68, 'label': 'flower', 'bbox': [0.2, 0.21, 0.45, 0.5},
            {'score': 0.78, 'label': 'flower', 'bbox': [0.3, 0.35, 0.48, 0.52},
            {'score': 0.98, 'label': 'flower', 'bbox': [0.44, 0.24, 0.49, 0.58},
        ]
    """
    image_size = image.shape[:2]
    if image_size[0] < 1 or image_size[1] < 1:
        return []

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    payload = {
        "prompts": [prompt.replace(", ", ". ")],
        "confidence": box_threshold,  # still not being used in the API
        "model": "countgd",
    }
    metadata = {"function_name": "countgd_counting"}

    detections = send_task_inference_request(
        payload, "text-to-object-detection", files=files, metadata=metadata
    )

    # get the first frame
    bboxes = detections[0]
    bboxes_formatted = [
        ODResponseData(
            label=bbox["label"],
            bbox=normalize_bbox(bbox["bounding_box"], image_size),
            score=round(bbox["score"], 2),
        )
        for bbox in bboxes
    ]
    # TODO: remove this once we start to use the confidence on countgd
    filtered_bboxes = filter_bboxes_by_threshold(bboxes_formatted, box_threshold)
    return_data = [bbox.model_dump() for bbox in filtered_bboxes]
    return single_nms(return_data, iou_threshold=0.80)

countgd_example_based_counting

countgd_example_based_counting(
    visual_prompts, image, box_threshold=0.23
)

'countgd_example_based_counting' is a tool that can precisely count multiple instances of an object given few visual example prompts. It returns a list of bounding boxes with normalized coordinates, label names and associated confidence scores.

PARAMETER	DESCRIPTION
visual_prompts	Bounding boxes of the object in format [xmin, ymin, xmax, ymax]. Upto 3 bounding boxes can be provided. image (np.ndarray): The image that contains multiple instances of the object. box_threshold (float, optional): The threshold for detection. Defaults to 0.23. TYPE: List[List[float]]

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> countgd_example_based_counting(
    visual_prompts=[[0.1, 0.1, 0.4, 0.42], [0.2, 0.3, 0.25, 0.35]],
    image=image
)
[
    {'score': 0.49, 'label': 'object', 'bounding_box': [0.1, 0.11, 0.35, 0.4]},
    {'score': 0.68, 'label': 'object', 'bounding_box': [0.2, 0.21, 0.45, 0.5},
    {'score': 0.78, 'label': 'object', 'bounding_box': [0.3, 0.35, 0.48, 0.52},
    {'score': 0.98, 'label': 'object', 'bounding_box': [0.44, 0.24, 0.49, 0.58},
]

Source code in vision_agent/tools/tools.py

def countgd_example_based_counting(
    visual_prompts: List[List[float]],
    image: np.ndarray,
    box_threshold: float = 0.23,
) -> List[Dict[str, Any]]:
    """'countgd_example_based_counting' is a tool that can precisely count multiple
    instances of an object given few visual example prompts. It returns a list of bounding
    boxes with normalized coordinates, label names and associated confidence scores.

    Parameters:
        visual_prompts (List[List[float]]): Bounding boxes of the object in format
            [xmin, ymin, xmax, ymax]. Upto 3 bounding boxes can be provided. image
            (np.ndarray): The image that contains multiple instances of the object.
            box_threshold (float, optional): The threshold for detection. Defaults to
            0.23.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label, and
            bounding box of the detected objects with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box.

    Example
    -------
        >>> countgd_example_based_counting(
            visual_prompts=[[0.1, 0.1, 0.4, 0.42], [0.2, 0.3, 0.25, 0.35]],
            image=image
        )
        [
            {'score': 0.49, 'label': 'object', 'bounding_box': [0.1, 0.11, 0.35, 0.4]},
            {'score': 0.68, 'label': 'object', 'bounding_box': [0.2, 0.21, 0.45, 0.5},
            {'score': 0.78, 'label': 'object', 'bounding_box': [0.3, 0.35, 0.48, 0.52},
            {'score': 0.98, 'label': 'object', 'bounding_box': [0.44, 0.24, 0.49, 0.58},
        ]
    """
    image_size = image.shape[:2]
    if image_size[0] < 1 or image_size[1] < 1:
        return []

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    visual_prompts = [
        denormalize_bbox(bbox, image.shape[:2]) for bbox in visual_prompts
    ]
    payload = {"visual_prompts": json.dumps(visual_prompts), "model": "countgd"}
    metadata = {"function_name": "countgd_example_based_counting"}

    detections = send_task_inference_request(
        payload, "visual-prompts-to-object-detection", files=files, metadata=metadata
    )

    # get the first frame
    bboxes_per_frame = detections[0]
    bboxes_formatted = [
        ODResponseData(
            label=bbox["label"],
            bbox=normalize_bbox(bbox["bounding_box"], image_size),
            score=round(bbox["score"], 2),
        )
        for bbox in bboxes_per_frame
    ]
    filtered_bboxes = filter_bboxes_by_threshold(bboxes_formatted, box_threshold)
    return [bbox.model_dump() for bbox in filtered_bboxes]

florence2_roberta_vqa

florence2_roberta_vqa(prompt, image)

'florence2_roberta_vqa' is a tool that takes an image and analyzes its contents, generates detailed captions and then tries to answer the given question using the generated context. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the image TYPE: str
image	The reference image used for the question TYPE: ndarray

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> florence2_roberta_vqa('What is the top left animal in this image?', image)
'white tiger'

Source code in vision_agent/tools/tools.py

def florence2_roberta_vqa(prompt: str, image: np.ndarray) -> str:
    """'florence2_roberta_vqa' is a tool that takes an image and analyzes
    its contents, generates detailed captions and then tries to answer the given
    question using the generated context. It returns text as an answer to the question.

    Parameters:
        prompt (str): The question about the image
        image (np.ndarray): The reference image used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> florence2_roberta_vqa('What is the top left animal in this image?', image)
        'white tiger'
    """

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "question": prompt,
        "function_name": "florence2_roberta_vqa",
    }

    answer = send_inference_request(data, "florence2-qa", v2=True)
    return answer  # type: ignore

ixc25_image_vqa

ixc25_image_vqa(prompt, image)

'ixc25_image_vqa' is a tool that can answer any questions about arbitrary images including regular images or images of documents or presentations. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the image TYPE: str
image	The reference image used for the question TYPE: ndarray

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> ixc25_image_vqa('What is the cat doing?', image)
'drinking milk'

Source code in vision_agent/tools/tools.py

def ixc25_image_vqa(prompt: str, image: np.ndarray) -> str:
    """'ixc25_image_vqa' is a tool that can answer any questions about arbitrary images
    including regular images or images of documents or presentations. It returns text
    as an answer to the question.

    Parameters:
        prompt (str): The question about the image
        image (np.ndarray): The reference image used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> ixc25_image_vqa('What is the cat doing?', image)
        'drinking milk'
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        raise ValueError(f"Image is empty, image shape: {image.shape}")

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    payload = {
        "prompt": prompt,
        "function_name": "ixc25_image_vqa",
    }
    data: Dict[str, Any] = send_inference_request(
        payload, "internlm-xcomposer2", files=files, v2=True
    )
    return cast(str, data["answer"])

qwen2_vl_images_vqa

qwen2_vl_images_vqa(prompt, images)

'qwen2_vl_images_vqa' is a tool that can answer any questions about arbitrary images including regular images or images of documents or presentations. It can be very useful for document QA or OCR text extraction. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the document image TYPE: str
images	The reference images used for the question TYPE: List[ndarray]

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> qwen2_vl_images_vqa('Give a summary of the document', images)
'The document talks about the history of the United States of America and its...'

Source code in vision_agent/tools/tools.py

def qwen2_vl_images_vqa(prompt: str, images: List[np.ndarray]) -> str:
    """'qwen2_vl_images_vqa' is a tool that can answer any questions about arbitrary
    images including regular images or images of documents or presentations. It can be
    very useful for document QA or OCR text extraction. It returns text as an answer to
    the question.

    Parameters:
        prompt (str): The question about the document image
        images (List[np.ndarray]): The reference images used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> qwen2_vl_images_vqa('Give a summary of the document', images)
        'The document talks about the history of the United States of America and its...'
    """
    if isinstance(images, np.ndarray):
        images = [images]

    for image in images:
        if image.shape[0] < 1 or image.shape[1] < 1:
            raise ValueError(f"Image is empty, image shape: {image.shape}")

    files = [("images", numpy_to_bytes(image)) for image in images]
    payload = {
        "prompt": prompt,
        "model": "qwen2vl",
        "function_name": "qwen2_vl_images_vqa",
    }
    data: Dict[str, Any] = send_inference_request(
        payload, "image-to-text", files=files, v2=True
    )
    return cast(str, data)

claude35_text_extraction

claude35_text_extraction(image)

'claude35_text_extraction' is a tool that can extract text from an image. It returns the extracted text as a string and can be used as an alternative to OCR if you do not need to know the exact bounding box of the text.

PARAMETER	DESCRIPTION
image	The image to extract text from. TYPE: ndarray

RETURNS	DESCRIPTION
str	The extracted text from the image. TYPE: str

Source code in vision_agent/tools/tools.py

def claude35_text_extraction(image: np.ndarray) -> str:
    """'claude35_text_extraction' is a tool that can extract text from an image. It
    returns the extracted text as a string and can be used as an alternative to OCR if
    you do not need to know the exact bounding box of the text.

    Parameters:
        image (np.ndarray): The image to extract text from.

    Returns:
        str: The extracted text from the image.
    """

    lmm = AnthropicLMM()
    buffer = io.BytesIO()
    Image.fromarray(image).save(buffer, format="PNG")
    image_bytes = buffer.getvalue()
    image_b64 = "data:image/png;base64," + encode_image_bytes(image_bytes)
    text = lmm.generate(
        "Extract and return any text you see in this image and nothing else. If you do not read any text respond with an empty string.",
        [image_b64],
    )
    return cast(str, text)

ixc25_video_vqa

ixc25_video_vqa(prompt, frames)

'ixc25_video_vqa' is a tool that can answer any questions about arbitrary videos including regular videos or videos of documents or presentations. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the video TYPE: str
frames	The reference frames used for the question TYPE: List[ndarray]

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> ixc25_video_vqa('Which football player made the goal?', frames)
'Lionel Messi'

Source code in vision_agent/tools/tools.py

def ixc25_video_vqa(prompt: str, frames: List[np.ndarray]) -> str:
    """'ixc25_video_vqa' is a tool that can answer any questions about arbitrary videos
    including regular videos or videos of documents or presentations. It returns text
    as an answer to the question.

    Parameters:
        prompt (str): The question about the video
        frames (List[np.ndarray]): The reference frames used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> ixc25_video_vqa('Which football player made the goal?', frames)
        'Lionel Messi'
    """

    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload = {
        "prompt": prompt,
        "function_name": "ixc25_video_vqa",
    }
    data: Dict[str, Any] = send_inference_request(
        payload, "internlm-xcomposer2", files=files, v2=True
    )
    return cast(str, data["answer"])

qwen2_vl_video_vqa

qwen2_vl_video_vqa(prompt, frames)

'qwen2_vl_video_vqa' is a tool that can answer any questions about arbitrary videos including regular videos or videos of documents or presentations. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the video TYPE: str
frames	The reference frames used for the question TYPE: List[ndarray]

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> qwen2_vl_video_vqa('Which football player made the goal?', frames)
'Lionel Messi'

Source code in vision_agent/tools/tools.py

def qwen2_vl_video_vqa(prompt: str, frames: List[np.ndarray]) -> str:
    """'qwen2_vl_video_vqa' is a tool that can answer any questions about arbitrary videos
    including regular videos or videos of documents or presentations. It returns text
    as an answer to the question.

    Parameters:
        prompt (str): The question about the video
        frames (List[np.ndarray]): The reference frames used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> qwen2_vl_video_vqa('Which football player made the goal?', frames)
        'Lionel Messi'
    """

    if len(frames) == 0 or not isinstance(frames, List):
        raise ValueError("Must provide a list of numpy arrays for frames")

    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload = {
        "prompt": prompt,
        "model": "qwen2vl",
        "function_name": "qwen2_vl_video_vqa",
    }
    data: Dict[str, Any] = send_inference_request(
        payload, "image-to-text", files=files, v2=True
    )
    return cast(str, data)

gpt4o_image_vqa

gpt4o_image_vqa(prompt, image)

'gpt4o_image_vqa' is a tool that can answer any questions about arbitrary images including regular images or images of documents or presentations. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the image TYPE: str
image	The reference image used for the question TYPE: ndarray

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> gpt4o_image_vqa('What is the cat doing?', image)
'drinking milk'

Source code in vision_agent/tools/tools.py

def gpt4o_image_vqa(prompt: str, image: np.ndarray) -> str:
    """'gpt4o_image_vqa' is a tool that can answer any questions about arbitrary images
    including regular images or images of documents or presentations. It returns text
    as an answer to the question.

    Parameters:
        prompt (str): The question about the image
        image (np.ndarray): The reference image used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> gpt4o_image_vqa('What is the cat doing?', image)
        'drinking milk'
    """

    lmm = OpenAILMM()
    buffer = io.BytesIO()
    Image.fromarray(image).save(buffer, format="PNG")
    image_bytes = buffer.getvalue()
    image_b64 = "data:image/png;base64," + encode_image_bytes(image_bytes)
    resp = lmm.generate(prompt, [image_b64])
    return cast(str, resp)

gpt4o_video_vqa

gpt4o_video_vqa(prompt, frames)

'gpt4o_video_vqa' is a tool that can answer any questions about arbitrary videos including regular videos or videos of documents or presentations. It returns text as an answer to the question.

PARAMETER	DESCRIPTION
prompt	The question about the video TYPE: str
frames	The reference frames used for the question TYPE: List[ndarray]

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> gpt4o_video_vqa('Which football player made the goal?', frames)
'Lionel Messi'

Source code in vision_agent/tools/tools.py

def gpt4o_video_vqa(prompt: str, frames: List[np.ndarray]) -> str:
    """'gpt4o_video_vqa' is a tool that can answer any questions about arbitrary videos
    including regular videos or videos of documents or presentations. It returns text
    as an answer to the question.

    Parameters:
        prompt (str): The question about the video
        frames (List[np.ndarray]): The reference frames used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> gpt4o_video_vqa('Which football player made the goal?', frames)
        'Lionel Messi'
    """

    lmm = OpenAILMM()

    if len(frames) > 10:
        step = len(frames) / 10
        frames = [frames[int(i * step)] for i in range(10)]

    frames_b64 = []
    for frame in frames:
        buffer = io.BytesIO()
        Image.fromarray(frame).save(buffer, format="PNG")
        image_bytes = buffer.getvalue()
        image_b64 = "data:image/png;base64," + encode_image_bytes(image_bytes)
        frames_b64.append(image_b64)

    resp = lmm.generate(prompt, frames_b64)
    return cast(str, resp)

git_vqa_v2

git_vqa_v2(prompt, image)

'git_vqa_v2' is a tool that can answer questions about the visual contents of an image given a question and an image. It returns an answer to the question

PARAMETER	DESCRIPTION
prompt	The question about the image TYPE: str
image	The reference image used for the question TYPE: ndarray

RETURNS	DESCRIPTION
str	A string which is the answer to the given prompt. TYPE: str

Example

>>> git_vqa_v2('What is the cat doing ?', image)
'drinking milk'

Source code in vision_agent/tools/tools.py

def git_vqa_v2(prompt: str, image: np.ndarray) -> str:
    """'git_vqa_v2' is a tool that can answer questions about the visual
    contents of an image given a question and an image. It returns an answer to the
    question

    Parameters:
        prompt (str): The question about the image
        image (np.ndarray): The reference image used for the question

    Returns:
        str: A string which is the answer to the given prompt.

    Example
    -------
        >>> git_vqa_v2('What is the cat doing ?', image)
        'drinking milk'
    """

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "prompt": prompt,
        "tool": "image_question_answering",
        "function_name": "git_vqa_v2",
    }

    answer = send_inference_request(data, "tools")
    return answer["text"][0]  # type: ignore

video_temporal_localization

video_temporal_localization(
    prompt, frames, model="qwen2vl", chunk_length_frames=2
)

'video_temporal_localization' will run qwen2vl on each chunk_length_frames value selected for the video. It can detect multiple objects independently per chunk_length_frames given a text prompt such as a referring expression but does not track objects across frames. It returns a list of floats with a value of 1.0 if the objects are found in a given chunk_length_frames of the video.

PARAMETER	DESCRIPTION
prompt	The question about the video TYPE: str
frames	The reference frames used for the question TYPE: List[ndarray]
model	The model to use for the inference. Valid values are 'qwen2vl', 'gpt4o', 'internlm-xcomposer' TYPE: str DEFAULT: 'qwen2vl'
chunk_length_frames	length of each chunk in frames TYPE: Optional[int] DEFAULT: 2

RETURNS	DESCRIPTION
List[float]	List[float]: A list of floats with a value of 1.0 if the objects to be found are present in the chunk_length_frames of the video.

Example

>>> video_temporal_localization('Did a goal happened?', frames)
[0.0, 0.0, 0.0, 1.0, 1.0, 0.0]

Source code in vision_agent/tools/tools.py

def video_temporal_localization(
    prompt: str,
    frames: List[np.ndarray],
    model: str = "qwen2vl",
    chunk_length_frames: Optional[int] = 2,
) -> List[float]:
    """'video_temporal_localization' will run qwen2vl on each chunk_length_frames
    value selected for the video. It can detect multiple objects independently per
    chunk_length_frames given a text prompt such as a referring expression
    but does not track objects across frames.
    It returns a list of floats with a value of 1.0 if the objects are found in a given
    chunk_length_frames of the video.

    Parameters:
        prompt (str): The question about the video
        frames (List[np.ndarray]): The reference frames used for the question
        model (str): The model to use for the inference. Valid values are
            'qwen2vl', 'gpt4o', 'internlm-xcomposer'
        chunk_length_frames (Optional[int]): length of each chunk in frames

    Returns:
        List[float]: A list of floats with a value of 1.0 if the objects to be found
            are present in the chunk_length_frames of the video.

    Example
    -------
        >>> video_temporal_localization('Did a goal happened?', frames)
        [0.0, 0.0, 0.0, 1.0, 1.0, 0.0]
    """

    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload: Dict[str, Any] = {
        "prompt": prompt,
        "model": model,
        "function_name": "video_temporal_localization",
    }
    if chunk_length_frames is not None:
        payload["chunk_length_frames"] = chunk_length_frames

    data = send_inference_request(
        payload, "video-temporal-localization", files=files, v2=True
    )
    return [cast(float, value) for value in data]

clip

clip(image, classes)

'clip' is a tool that can classify an image or a cropped detection given a list of input classes or tags. It returns the same list of the input classes along with their probability scores based on image content.

PARAMETER	DESCRIPTION
image	The image to classify or tag TYPE: ndarray
classes	The list of classes or tags that is associated with the image TYPE: List[str]

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the labels and scores. One dictionary contains a list of given labels and other a list of scores.

Example

>>> clip(image, ['dog', 'cat', 'bird'])
{"labels": ["dog", "cat", "bird"], "scores": [0.68, 0.30, 0.02]},

Source code in vision_agent/tools/tools.py

def clip(image: np.ndarray, classes: List[str]) -> Dict[str, Any]:
    """'clip' is a tool that can classify an image or a cropped detection given a list
    of input classes or tags. It returns the same list of the input classes along with
    their probability scores based on image content.

    Parameters:
        image (np.ndarray): The image to classify or tag
        classes (List[str]): The list of classes or tags that is associated with the image

    Returns:
        Dict[str, Any]: A dictionary containing the labels and scores. One dictionary
            contains a list of given labels and other a list of scores.

    Example
    -------
        >>> clip(image, ['dog', 'cat', 'bird'])
        {"labels": ["dog", "cat", "bird"], "scores": [0.68, 0.30, 0.02]},
    """

    if image.shape[0] < 1 or image.shape[1] < 1:
        return {"labels": [], "scores": []}

    image_b64 = convert_to_b64(image)
    data = {
        "prompt": ",".join(classes),
        "image": image_b64,
        "tool": "closed_set_image_classification",
        "function_name": "clip",
    }
    resp_data: dict[str, Any] = send_inference_request(data, "tools")
    resp_data["scores"] = [round(prob, 4) for prob in resp_data["scores"]]
    return resp_data

vit_image_classification

vit_image_classification(image)

'vit_image_classification' is a tool that can classify an image. It returns a list of classes and their probability scores based on image content.

PARAMETER	DESCRIPTION
image	The image to classify or tag TYPE: ndarray

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the labels and scores. One dictionary contains a list of labels and other a list of scores.

Example

>>> vit_image_classification(image)
{"labels": ["leopard", "lemur, otter", "bird"], "scores": [0.68, 0.30, 0.02]},

Source code in vision_agent/tools/tools.py

def vit_image_classification(image: np.ndarray) -> Dict[str, Any]:
    """'vit_image_classification' is a tool that can classify an image. It returns a
    list of classes and their probability scores based on image content.

    Parameters:
        image (np.ndarray): The image to classify or tag

    Returns:
        Dict[str, Any]: A dictionary containing the labels and scores. One dictionary
            contains a list of labels and other a list of scores.

    Example
    -------
        >>> vit_image_classification(image)
        {"labels": ["leopard", "lemur, otter", "bird"], "scores": [0.68, 0.30, 0.02]},
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        return {"labels": [], "scores": []}

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "tool": "image_classification",
        "function_name": "vit_image_classification",
    }
    resp_data: dict[str, Any] = send_inference_request(data, "tools")
    resp_data["scores"] = [round(prob, 4) for prob in resp_data["scores"]]
    return resp_data

vit_nsfw_classification

vit_nsfw_classification(image)

'vit_nsfw_classification' is a tool that can classify an image as 'nsfw' or 'normal'. It returns the predicted label and their probability scores based on image content.

PARAMETER	DESCRIPTION
image	The image to classify or tag TYPE: ndarray

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the labels and scores. One dictionary contains a list of labels and other a list of scores.

Example

>>> vit_nsfw_classification(image)
{"label": "normal", "scores": 0.68},

Source code in vision_agent/tools/tools.py

def vit_nsfw_classification(image: np.ndarray) -> Dict[str, Any]:
    """'vit_nsfw_classification' is a tool that can classify an image as 'nsfw' or 'normal'.
    It returns the predicted label and their probability scores based on image content.

    Parameters:
        image (np.ndarray): The image to classify or tag

    Returns:
        Dict[str, Any]: A dictionary containing the labels and scores. One dictionary
            contains a list of labels and other a list of scores.

    Example
    -------
        >>> vit_nsfw_classification(image)
        {"label": "normal", "scores": 0.68},
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        raise ValueError(f"Image is empty, image shape: {image.shape}")

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "function_name": "vit_nsfw_classification",
    }
    resp_data: dict[str, Any] = send_inference_request(
        data, "nsfw-classification", v2=True
    )
    resp_data["score"] = round(resp_data["score"], 4)
    return resp_data

blip_image_caption

blip_image_caption(image)

'blip_image_caption' is a tool that can caption an image based on its contents. It returns a text describing the image.

PARAMETER	DESCRIPTION
image	The image to caption TYPE: ndarray

RETURNS	DESCRIPTION
str	A string which is the caption for the given image. TYPE: str

Example

>>> blip_image_caption(image)
'This image contains a cat sitting on a table with a bowl of milk.'

Source code in vision_agent/tools/tools.py

def blip_image_caption(image: np.ndarray) -> str:
    """'blip_image_caption' is a tool that can caption an image based on its contents. It
    returns a text describing the image.

    Parameters:
        image (np.ndarray): The image to caption

    Returns:
       str: A string which is the caption for the given image.

    Example
    -------
        >>> blip_image_caption(image)
        'This image contains a cat sitting on a table with a bowl of milk.'
    """

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "tool": "image_captioning",
        "function_name": "blip_image_caption",
    }

    answer = send_inference_request(data, "tools")
    return answer["text"][0]  # type: ignore

florence2_image_caption

florence2_image_caption(image, detail_caption=True)

'florence2_image_caption' is a tool that can caption or describe an image based on its contents. It returns a text describing the image.

PARAMETER	DESCRIPTION
image	The image to caption TYPE: ndarray
detail_caption	If True, the caption will be as detailed as possible else the caption will be a brief description. TYPE: bool DEFAULT: True

RETURNS	DESCRIPTION
str	A string which is the caption for the given image. TYPE: str

Example

>>> florence2_image_caption(image, False)
'This image contains a cat sitting on a table with a bowl of milk.'

Source code in vision_agent/tools/tools.py

def florence2_image_caption(image: np.ndarray, detail_caption: bool = True) -> str:
    """'florence2_image_caption' is a tool that can caption or describe an image based
    on its contents. It returns a text describing the image.

    Parameters:
        image (np.ndarray): The image to caption
        detail_caption (bool): If True, the caption will be as detailed as possible else
            the caption will be a brief description.

    Returns:
       str: A string which is the caption for the given image.

    Example
    -------
        >>> florence2_image_caption(image, False)
        'This image contains a cat sitting on a table with a bowl of milk.'
    """
    image_b64 = convert_to_b64(image)
    task = "<MORE_DETAILED_CAPTION>" if detail_caption else "<DETAILED_CAPTION>"
    data = {
        "image": image_b64,
        "task": task,
        "function_name": "florence2_image_caption",
    }

    answer = send_inference_request(data, "florence2", v2=True)
    return answer[task]  # type: ignore

florence2_phrase_grounding

florence2_phrase_grounding(
    prompt, image, fine_tune_id=None
)

'florence2_phrase_grounding' is a tool that can detect multiple objects given a text prompt which can be object names or caption. You can optionally separate the object names in the text with commas. It returns a list of bounding boxes with normalized coordinates, label names and associated probability scores of 1.0.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the image. TYPE: str
image	The image to used to detect objects TYPE: ndarray
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box. The scores are always 1.0 and cannot be thresholded

Example

>>> florence2_phrase_grounding('person looking at a coyote', image)
[
    {'score': 1.0, 'label': 'person', 'bbox': [0.1, 0.11, 0.35, 0.4]},
    {'score': 1.0, 'label': 'coyote', 'bbox': [0.34, 0.21, 0.85, 0.5},
]

Source code in vision_agent/tools/tools.py

def florence2_phrase_grounding(
    prompt: str, image: np.ndarray, fine_tune_id: Optional[str] = None
) -> List[Dict[str, Any]]:
    """'florence2_phrase_grounding' is a tool that can detect multiple
    objects given a text prompt which can be object names or caption. You
    can optionally separate the object names in the text with commas. It returns a list
    of bounding boxes with normalized coordinates, label names and associated
    probability scores of 1.0.

    Parameters:
        prompt (str): The prompt to ground to the image.
        image (np.ndarray): The image to used to detect objects
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label, and
            bounding box of the detected objects with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box. The scores are always 1.0 and cannot be thresholded

    Example
    -------
        >>> florence2_phrase_grounding('person looking at a coyote', image)
        [
            {'score': 1.0, 'label': 'person', 'bbox': [0.1, 0.11, 0.35, 0.4]},
            {'score': 1.0, 'label': 'coyote', 'bbox': [0.34, 0.21, 0.85, 0.5},
        ]
    """
    image_size = image.shape[:2]
    if image_size[0] < 1 or image_size[1] < 1:
        return []

    buffer_bytes = numpy_to_bytes(image)
    files = [("image", buffer_bytes)]
    payload = {
        "prompts": [s.strip() for s in prompt.split(",")],
        "model": "florence2",
    }
    metadata = {"function_name": "florence2_phrase_grounding"}

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        payload["jobId"] = fine_tune_id

    detections = send_task_inference_request(
        payload,
        "text-to-object-detection",
        files=files,
        metadata=metadata,
    )

    # get the first frame
    bboxes = detections[0]
    bboxes_formatted = [
        ODResponseData(
            label=bbox["label"],
            bbox=normalize_bbox(bbox["bounding_box"], image_size),
            score=round(bbox["score"], 2),
        )
        for bbox in bboxes
    ]

    return [bbox.model_dump() for bbox in bboxes_formatted]

florence2_phrase_grounding_video

florence2_phrase_grounding_video(
    prompt, frames, fine_tune_id=None
)

'florence2_phrase_grounding_video' will run florence2 on each frame of a video. It can detect multiple objects given a text prompt which can be object names or caption. You can optionally separate the object names in the text with commas. It returns a list of lists where each inner list contains bounding boxes with normalized coordinates, label names and associated probability scores of 1.0.

PARAMETER	DESCRIPTION
prompt	The prompt to ground to the video. TYPE: str
frames	The list of frames to detect objects. TYPE: List[ndarray]
fine_tune_id	If you have a fine-tuned model, you can pass the fine-tuned model ID here to use it. TYPE: Optional[str] DEFAULT: None

RETURNS	DESCRIPTION
List[List[Dict[str, Any]]]	List[List[Dict[str, Any]]]: A list of lists of dictionaries containing the score, label, and bounding box of the detected objects with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box. The scores are always 1.0 and cannot be thresholded.

Example

>>> florence2_phrase_grounding_video('person looking at a coyote', frames)
[
    [
        {'score': 1.0, 'label': 'person', 'bbox': [0.1, 0.11, 0.35, 0.4]},
        {'score': 1.0, 'label': 'coyote', 'bbox': [0.34, 0.21, 0.85, 0.5},
    ],
    ...
]

Source code in vision_agent/tools/tools.py

def florence2_phrase_grounding_video(
    prompt: str, frames: List[np.ndarray], fine_tune_id: Optional[str] = None
) -> List[List[Dict[str, Any]]]:
    """'florence2_phrase_grounding_video' will run florence2 on each frame of a video.
    It can detect multiple objects given a text prompt which can be object names or
    caption. You can optionally separate the object names in the text with commas.
    It returns a list of lists where each inner list contains bounding boxes with
    normalized coordinates, label names and associated probability scores of 1.0.

    Parameters:
        prompt (str): The prompt to ground to the video.
        frames (List[np.ndarray]): The list of frames to detect objects.
        fine_tune_id (Optional[str]): If you have a fine-tuned model, you can pass the
            fine-tuned model ID here to use it.

    Returns:
        List[List[Dict[str, Any]]]: A list of lists of dictionaries containing the score,
            label, and bounding box of the detected objects with normalized coordinates
            between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates
            of the top-left and xmax and ymax are the coordinates of the bottom-right of
            the bounding box. The scores are always 1.0 and cannot be thresholded.

    Example
    -------
        >>> florence2_phrase_grounding_video('person looking at a coyote', frames)
        [
            [
                {'score': 1.0, 'label': 'person', 'bbox': [0.1, 0.11, 0.35, 0.4]},
                {'score': 1.0, 'label': 'coyote', 'bbox': [0.34, 0.21, 0.85, 0.5},
            ],
            ...
        ]
    """
    if len(frames) == 0:
        raise ValueError("No frames provided")

    image_size = frames[0].shape[:2]
    buffer_bytes = frames_to_bytes(frames)
    files = [("video", buffer_bytes)]
    payload = {
        "prompts": [s.strip() for s in prompt.split(",")],
        "model": "florence2",
    }
    metadata = {"function_name": "florence2_phrase_grounding_video"}

    if fine_tune_id is not None:
        landing_api = LandingPublicAPI()
        status = landing_api.check_fine_tuning_job(UUID(fine_tune_id))
        if status is not JobStatus.SUCCEEDED:
            raise FineTuneModelIsNotReady(
                f"Fine-tuned model {fine_tune_id} is not ready yet"
            )

        payload["jobId"] = fine_tune_id

    detections = send_task_inference_request(
        payload,
        "text-to-object-detection",
        files=files,
        metadata=metadata,
    )

    bboxes_formatted = []
    for frame_data in detections:
        bboxes_formatted_per_frame = [
            ODResponseData(
                label=bbox["label"],
                bbox=normalize_bbox(bbox["bounding_box"], image_size),
                score=round(bbox["score"], 2),
            )
            for bbox in frame_data
        ]
        bboxes_formatted.append(bboxes_formatted_per_frame)
    return [[bbox.model_dump() for bbox in frame] for frame in bboxes_formatted]

florence2_ocr

florence2_ocr(image)

'florence2_ocr' is a tool that can detect text and text regions in an image. Each text region contains one line of text. It returns a list of detected text, the text region as a bounding box with normalized coordinates, and confidence scores. The results are sorted from top-left to bottom right.

PARAMETER	DESCRIPTION
image	The image to extract text from. TYPE: ndarray

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the detected text, bbox with normalized coordinates, and confidence score.

Example

>>> florence2_ocr(image)
[
    {'label': 'hello world', 'bbox': [0.1, 0.11, 0.35, 0.4], 'score': 0.99},
]

Source code in vision_agent/tools/tools.py

def florence2_ocr(image: np.ndarray) -> List[Dict[str, Any]]:
    """'florence2_ocr' is a tool that can detect text and text regions in an image.
    Each text region contains one line of text. It returns a list of detected text,
    the text region as a bounding box with normalized coordinates, and confidence
    scores. The results are sorted from top-left to bottom right.

    Parameters:
        image (np.ndarray): The image to extract text from.

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the detected text, bbox
            with normalized coordinates, and confidence score.

    Example
    -------
        >>> florence2_ocr(image)
        [
            {'label': 'hello world', 'bbox': [0.1, 0.11, 0.35, 0.4], 'score': 0.99},
        ]
    """

    image_size = image.shape[:2]
    if image_size[0] < 1 or image_size[1] < 1:
        return []
    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "task": "<OCR_WITH_REGION>",
        "function_name": "florence2_ocr",
    }

    detections = send_inference_request(data, "florence2", v2=True)
    detections = detections["<OCR_WITH_REGION>"]
    return_data = []
    for i in range(len(detections["quad_boxes"])):
        return_data.append(
            {
                "label": detections["labels"][i],
                "bbox": normalize_bbox(
                    convert_quad_box_to_bbox(detections["quad_boxes"][i]), image_size
                ),
                "score": 1.0,
            }
        )
    return return_data

detr_segmentation

detr_segmentation(image)

'detr_segmentation' is a tool that can segment common objects in an image without any text prompt. It returns a list of detected objects as labels, their regions as masks and their scores.

PARAMETER	DESCRIPTION
image	The image used to segment things and objects TYPE: ndarray

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score, label and mask of the detected objects. The mask is binary 2D numpy array where 1 indicates the object and 0 indicates the background.

Example

>>> detr_segmentation(image)
[
    {
        'score': 0.45,
        'label': 'window',
        'mask': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
    },
    {
        'score': 0.70,
        'label': 'bird',
        'mask': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
    },
]

Source code in vision_agent/tools/tools.py

def detr_segmentation(image: np.ndarray) -> List[Dict[str, Any]]:
    """'detr_segmentation' is a tool that can segment common objects in an
    image without any text prompt. It returns a list of detected objects
    as labels, their regions as masks and their scores.

    Parameters:
        image (np.ndarray): The image used to segment things and objects

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score, label
            and mask of the detected objects. The mask is binary 2D numpy array where 1
            indicates the object and 0 indicates the background.

    Example
    -------
        >>> detr_segmentation(image)
        [
            {
                'score': 0.45,
                'label': 'window',
                'mask': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
            },
            {
                'score': 0.70,
                'label': 'bird',
                'mask': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
            },
        ]
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        return []
    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "tool": "panoptic_segmentation",
        "function_name": "detr_segmentation",
    }

    answer = send_inference_request(data, "tools")
    return_data = []

    for i in range(len(answer["scores"])):
        return_data.append(
            {
                "score": round(answer["scores"][i], 2),
                "label": answer["labels"][i],
                "mask": rle_decode(
                    mask_rle=answer["masks"][i], shape=answer["mask_shape"][0]
                ),
            }
        )
    return return_data

depth_anything_v2

depth_anything_v2(image)

'depth_anything_v2' is a tool that runs depth_anythingv2 model to generate a depth image from a given RGB image. The returned depth image is monochrome and represents depth values as pixel intesities with pixel values ranging from 0 to 255.

PARAMETER	DESCRIPTION
image	The image to used to generate depth image TYPE: ndarray

RETURNS	DESCRIPTION
ndarray	np.ndarray: A grayscale depth image with pixel values ranging from 0 to 255.

Example

>>> depth_anything_v2(image)
array([[0, 0, 0, ..., 0, 0, 0],
        [0, 20, 24, ..., 0, 100, 103],
        ...,
        [10, 11, 15, ..., 202, 202, 205],
        [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),

Source code in vision_agent/tools/tools.py

def depth_anything_v2(image: np.ndarray) -> np.ndarray:
    """'depth_anything_v2' is a tool that runs depth_anythingv2 model to generate a
    depth image from a given RGB image. The returned depth image is monochrome and
    represents depth values as pixel intesities with pixel values ranging from 0 to 255.

    Parameters:
        image (np.ndarray): The image to used to generate depth image

    Returns:
        np.ndarray: A grayscale depth image with pixel values ranging from 0 to 255.

    Example
    -------
        >>> depth_anything_v2(image)
        array([[0, 0, 0, ..., 0, 0, 0],
                [0, 20, 24, ..., 0, 100, 103],
                ...,
                [10, 11, 15, ..., 202, 202, 205],
                [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),
    """
    if image.shape[0] < 1 or image.shape[1] < 1:
        raise ValueError(f"Image is empty, image shape: {image.shape}")

    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "function_name": "depth_anything_v2",
    }

    depth_map = send_inference_request(data, "depth-anything-v2", v2=True)
    depth_map_np = np.array(depth_map["map"])
    depth_map_np = (depth_map_np - depth_map_np.min()) / (
        depth_map_np.max() - depth_map_np.min()
    )
    depth_map_np = (255 * depth_map_np).astype(np.uint8)
    return depth_map_np

generate_soft_edge_image

generate_soft_edge_image(image)

'generate_soft_edge_image' is a tool that runs Holistically Nested edge detection to generate a soft edge image (HED) from a given RGB image. The returned image is monochrome and represents object boundaries as soft white edges on black background

PARAMETER	DESCRIPTION
image	The image to used to generate soft edge image TYPE: ndarray

RETURNS	DESCRIPTION
ndarray	np.ndarray: A soft edge image with pixel values ranging from 0 to 255.

Example

>>> generate_soft_edge_image(image)
array([[0, 0, 0, ..., 0, 0, 0],
        [0, 20, 24, ..., 0, 100, 103],
        ...,
        [10, 11, 15, ..., 202, 202, 205],
        [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),

Source code in vision_agent/tools/tools.py

def generate_soft_edge_image(image: np.ndarray) -> np.ndarray:
    """'generate_soft_edge_image' is a tool that runs Holistically Nested edge detection
    to generate a soft edge image (HED) from a given RGB image. The returned image is
    monochrome and represents object boundaries as soft white edges on black background

    Parameters:
        image (np.ndarray): The image to used to generate soft edge image

    Returns:
        np.ndarray: A soft edge image with pixel values ranging from 0 to 255.

    Example
    -------
        >>> generate_soft_edge_image(image)
        array([[0, 0, 0, ..., 0, 0, 0],
                [0, 20, 24, ..., 0, 100, 103],
                ...,
                [10, 11, 15, ..., 202, 202, 205],
                [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),
    """
    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "tool": "generate_hed",
        "function_name": "generate_soft_edge_image",
    }

    answer = send_inference_request(data, "tools")
    return_data = np.array(b64_to_pil(answer["masks"][0]).convert("L"))
    return return_data

dpt_hybrid_midas

dpt_hybrid_midas(image)

'dpt_hybrid_midas' is a tool that generates a normal mapped from a given RGB image. The returned RGB image is texture mapped image of the surface normals and the RGB values represent the surface normals in the x, y, z directions.

PARAMETER	DESCRIPTION
image	The image to used to generate normal image TYPE: ndarray

RETURNS	DESCRIPTION
ndarray	np.ndarray: A mapped normal image with RGB pixel values indicating surface
ndarray	normals in x, y, z directions.

Example

>>> dpt_hybrid_midas(image)
array([[0, 0, 0, ..., 0, 0, 0],
        [0, 20, 24, ..., 0, 100, 103],
        ...,
        [10, 11, 15, ..., 202, 202, 205],
        [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),

Source code in vision_agent/tools/tools.py

def dpt_hybrid_midas(image: np.ndarray) -> np.ndarray:
    """'dpt_hybrid_midas' is a tool that generates a normal mapped from a given RGB
    image. The returned RGB image is texture mapped image of the surface normals and the
    RGB values represent the surface normals in the x, y, z directions.

    Parameters:
        image (np.ndarray): The image to used to generate normal image

    Returns:
        np.ndarray: A mapped normal image with RGB pixel values indicating surface
        normals in x, y, z directions.

    Example
    -------
        >>> dpt_hybrid_midas(image)
        array([[0, 0, 0, ..., 0, 0, 0],
                [0, 20, 24, ..., 0, 100, 103],
                ...,
                [10, 11, 15, ..., 202, 202, 205],
                [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),
    """
    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "tool": "generate_normal",
        "function_name": "dpt_hybrid_midas",
    }

    answer = send_inference_request(data, "tools")
    return_data = np.array(b64_to_pil(answer["masks"][0]).convert("RGB"))
    return return_data

generate_pose_image

generate_pose_image(image)

'generate_pose_image' is a tool that generates a open pose bone/stick image from a given RGB image. The returned bone image is RGB with the pose amd keypoints colored and background as black.

PARAMETER	DESCRIPTION
image	The image to used to generate pose image TYPE: ndarray

RETURNS	DESCRIPTION
ndarray	np.ndarray: A bone or pose image indicating the pose and keypoints

Example

>>> generate_pose_image(image)
array([[0, 0, 0, ..., 0, 0, 0],
        [0, 20, 24, ..., 0, 100, 103],
        ...,
        [10, 11, 15, ..., 202, 202, 205],
        [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),

Source code in vision_agent/tools/tools.py

def generate_pose_image(image: np.ndarray) -> np.ndarray:
    """'generate_pose_image' is a tool that generates a open pose bone/stick image from
    a given RGB image. The returned bone image is RGB with the pose amd keypoints colored
    and background as black.

    Parameters:
        image (np.ndarray): The image to used to generate pose image

    Returns:
        np.ndarray: A bone or pose image indicating the pose and keypoints

    Example
    -------
        >>> generate_pose_image(image)
        array([[0, 0, 0, ..., 0, 0, 0],
                [0, 20, 24, ..., 0, 100, 103],
                ...,
                [10, 11, 15, ..., 202, 202, 205],
                [10, 10, 10, ..., 200, 200, 200]], dtype=uint8),
    """
    image_b64 = convert_to_b64(image)
    data = {
        "image": image_b64,
        "function_name": "generate_pose_image",
    }

    pos_img = send_inference_request(data, "pose-detector", v2=True)
    return_data = np.array(b64_to_pil(pos_img["data"]).convert("RGB"))
    return return_data

template_match

template_match(image, template_image)

'template_match' is a tool that can detect all instances of a template in a given image. It returns the locations of the detected template, a corresponding similarity score of the same

PARAMETER	DESCRIPTION
image	The image used for searching the template TYPE: ndarray
template_image	The template image or crop to search in the image TYPE: ndarray

RETURNS	DESCRIPTION
List[Dict[str, Any]]	List[Dict[str, Any]]: A list of dictionaries containing the score and bounding box of the detected template with normalized coordinates between 0 and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the top-left and xmax and ymax are the coordinates of the bottom-right of the bounding box.

Example

>>> template_match(image, template)
[
    {'score': 0.79, 'bbox': [0.1, 0.11, 0.35, 0.4]},
    {'score': 0.38, 'bbox': [0.2, 0.21, 0.45, 0.5},
]

Source code in vision_agent/tools/tools.py

def template_match(
    image: np.ndarray, template_image: np.ndarray
) -> List[Dict[str, Any]]:
    """'template_match' is a tool that can detect all instances of a template in
    a given image. It returns the locations of the detected template, a corresponding
    similarity score of the same

    Parameters:
        image (np.ndarray): The image used for searching the template
        template_image (np.ndarray): The template image or crop to search in the image

    Returns:
        List[Dict[str, Any]]: A list of dictionaries containing the score and
            bounding box of the detected template with normalized coordinates between 0
            and 1 (xmin, ymin, xmax, ymax). xmin and ymin are the coordinates of the
            top-left and xmax and ymax are the coordinates of the bottom-right of the
            bounding box.

    Example
    -------
        >>> template_match(image, template)
        [
            {'score': 0.79, 'bbox': [0.1, 0.11, 0.35, 0.4]},
            {'score': 0.38, 'bbox': [0.2, 0.21, 0.45, 0.5},
        ]
    """
    image_size = image.shape[:2]
    image_b64 = convert_to_b64(image)
    template_image_b64 = convert_to_b64(template_image)
    data = {
        "image": image_b64,
        "template": template_image_b64,
        "tool": "template_match",
        "function_name": "template_match",
    }

    answer = send_inference_request(data, "tools")
    return_data = []
    for i in range(len(answer["bboxes"])):
        return_data.append(
            {
                "score": round(answer["scores"][i], 2),
                "bbox": normalize_bbox(answer["bboxes"][i], image_size),
            }
        )
    return return_data

closest_mask_distance

closest_mask_distance(mask1, mask2)

'closest_mask_distance' calculates the closest distance between two masks.

PARAMETER	DESCRIPTION
mask1	The first mask. TYPE: ndarray
mask2	The second mask. TYPE: ndarray

RETURNS	DESCRIPTION
float	The closest distance between the two masks. TYPE: float

Example

>>> closest_mask_distance(mask1, mask2)
0.5

Source code in vision_agent/tools/tools.py

def closest_mask_distance(mask1: np.ndarray, mask2: np.ndarray) -> float:
    """'closest_mask_distance' calculates the closest distance between two masks.

    Parameters:
        mask1 (np.ndarray): The first mask.
        mask2 (np.ndarray): The second mask.

    Returns:
        float: The closest distance between the two masks.

    Example
    -------
        >>> closest_mask_distance(mask1, mask2)
        0.5
    """

    mask1 = np.clip(mask1, 0, 1)
    mask2 = np.clip(mask2, 0, 1)
    contours1, _ = cv2.findContours(mask1, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    contours2, _ = cv2.findContours(mask2, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    largest_contour1 = max(contours1, key=cv2.contourArea)
    largest_contour2 = max(contours2, key=cv2.contourArea)
    polygon1 = cv2.approxPolyDP(largest_contour1, 1.0, True)
    polygon2 = cv2.approxPolyDP(largest_contour2, 1.0, True)
    min_distance = np.inf

    small_polygon, larger_contour = (
        (polygon1, largest_contour2)
        if len(largest_contour1) < len(largest_contour2)
        else (polygon2, largest_contour1)
    )

    # For each point in the first polygon
    for point in small_polygon:
        # Calculate the distance to the second polygon, -1 is to invert result as point inside the polygon is positive

        distance = (
            cv2.pointPolygonTest(
                larger_contour, (point[0, 0].item(), point[0, 1].item()), True
            )
            * -1
        )

        # If the distance is negative, the point is inside the polygon, so the distance is 0
        if distance < 0:
            continue
        else:
            # Update the minimum distance if the point is outside the polygon
            min_distance = min(min_distance, distance)

    return min_distance if min_distance != np.inf else 0.0

closest_box_distance

closest_box_distance(box1, box2, image_size)

'closest_box_distance' calculates the closest distance between two bounding boxes.

PARAMETER	DESCRIPTION
box1	The first bounding box. TYPE: List[float]
box2	The second bounding box. TYPE: List[float]
image_size	The size of the image given as (height, width). TYPE: Tuple[int, int]

RETURNS	DESCRIPTION
float	The closest distance between the two bounding boxes. TYPE: float

Example

>>> closest_box_distance([100, 100, 200, 200], [300, 300, 400, 400])
141.42

Source code in vision_agent/tools/tools.py

def closest_box_distance(
    box1: List[float], box2: List[float], image_size: Tuple[int, int]
) -> float:
    """'closest_box_distance' calculates the closest distance between two bounding boxes.

    Parameters:
        box1 (List[float]): The first bounding box.
        box2 (List[float]): The second bounding box.
        image_size (Tuple[int, int]): The size of the image given as (height, width).

    Returns:
        float: The closest distance between the two bounding boxes.

    Example
    -------
        >>> closest_box_distance([100, 100, 200, 200], [300, 300, 400, 400])
        141.42
    """

    x11, y11, x12, y12 = denormalize_bbox(box1, image_size)
    x21, y21, x22, y22 = denormalize_bbox(box2, image_size)

    horizontal_distance = np.max([0, x21 - x12, x11 - x22])
    vertical_distance = np.max([0, y21 - y12, y11 - y22])
    return cast(float, np.sqrt(horizontal_distance**2 + vertical_distance**2))

flux_image_inpainting

flux_image_inpainting(prompt, image, mask)

'flux_image_inpainting' performs image inpainting to fill the masked regions, given by mask, in the image, given image based on the text prompt and surrounding image context. It can be used to edit regions of an image according to the prompt given.

PARAMETER	DESCRIPTION
prompt	A detailed text description guiding what should be generated in the masked area. More detailed and specific prompts typically yield better results. TYPE: str
image	The source image to be inpainted. The image will serve as the base context for the inpainting process. TYPE: ndarray
mask	A binary mask image with 0's and 1's, where 1 indicates areas to be inpainted and 0 indicates areas to be preserved. TYPE: ndarray

RETURNS	DESCRIPTION
ndarray	np.ndarray: The generated image(s) as a numpy array in RGB format with values ranging from 0 to 255.

---

Example: >>> # Generate inpainting >>> result = flux_image_inpainting( ... prompt="a modern black leather sofa with white pillows", ... image=image, ... mask=mask, ... ) >>> save_image(result, "inpainted_room.png")

Source code in vision_agent/tools/tools.py

def flux_image_inpainting(
    prompt: str,
    image: np.ndarray,
    mask: np.ndarray,
) -> np.ndarray:
    """'flux_image_inpainting' performs image inpainting to fill the masked regions,
    given by mask, in the image, given image based on the text prompt and surrounding image context.
    It can be used to edit regions of an image according to the prompt given.

    Parameters:
        prompt (str): A detailed text description guiding what should be generated
            in the masked area. More detailed and specific prompts typically yield better results.
        image (np.ndarray): The source image to be inpainted.
            The image will serve as the base context for the inpainting process.
        mask (np.ndarray): A binary mask image with 0's and 1's,
            where 1 indicates areas to be inpainted and 0 indicates areas to be preserved.

    Returns:
        np.ndarray: The generated image(s) as a numpy array in RGB format with values
            ranging from 0 to 255.

    -------
    Example:
        >>> # Generate inpainting
        >>> result = flux_image_inpainting(
        ...     prompt="a modern black leather sofa with white pillows",
        ...     image=image,
        ...     mask=mask,
        ... )
        >>> save_image(result, "inpainted_room.png")
    """

    min_dim = 8

    if any(dim < min_dim for dim in image.shape[:2] + mask.shape[:2]):
        raise ValueError(f"Image and mask must be at least {min_dim}x{min_dim} pixels")

    max_size = (512, 512)

    if image.shape[0] > max_size[0] or image.shape[1] > max_size[1]:
        scaling_factor = min(max_size[0] / image.shape[0], max_size[1] / image.shape[1])
        new_size = (
            int(image.shape[1] * scaling_factor),
            int(image.shape[0] * scaling_factor),
        )
        new_size = ((new_size[0] // 8) * 8, (new_size[1] // 8) * 8)
        image = cv2.resize(image, new_size, interpolation=cv2.INTER_AREA)
        mask = cv2.resize(mask, new_size, interpolation=cv2.INTER_NEAREST)

    elif image.shape[0] % 8 != 0 or image.shape[1] % 8 != 0:
        new_size = ((image.shape[1] // 8) * 8, (image.shape[0] // 8) * 8)
        image = cv2.resize(image, new_size, interpolation=cv2.INTER_AREA)
        mask = cv2.resize(mask, new_size, interpolation=cv2.INTER_NEAREST)

    if np.array_equal(mask, mask.astype(bool).astype(int)):
        mask = np.where(mask > 0, 255, 0).astype(np.uint8)
    else:
        raise ValueError("Mask should contain only binary values (0 or 1)")

    image_file = numpy_to_bytes(image)
    mask_file = numpy_to_bytes(mask)

    files = [
        ("image", image_file),
        ("mask_image", mask_file),
    ]

    payload = {
        "prompt": prompt,
        "task": "inpainting",
        "height": image.shape[0],
        "width": image.shape[1],
        "strength": 0.99,
        "guidance_scale": 18,
        "num_inference_steps": 20,
        "seed": None,
    }

    response = send_inference_request(
        payload=payload,
        endpoint_name="flux1",
        files=files,
        v2=True,
        metadata_payload={"function_name": "flux_image_inpainting"},
    )

    output_image = np.array(b64_to_pil(response[0]).convert("RGB"))
    return output_image

siglip_classification

siglip_classification(image, labels)

'siglip_classification' is a tool that can classify an image or a cropped detection given a list of input labels or tags. It returns the same list of the input labels along with their probability scores based on image content.

PARAMETER	DESCRIPTION
image	The image to classify or tag TYPE: ndarray
labels	The list of labels or tags that is associated with the image TYPE: List[str]

RETURNS	DESCRIPTION
Dict[str, Any]	Dict[str, Any]: A dictionary containing the labels and scores. One dictionary contains a list of given labels and other a list of scores.

Example

>>> siglip_classification(image, ['dog', 'cat', 'bird'])
{"labels": ["dog", "cat", "bird"], "scores": [0.68, 0.30, 0.02]},

Source code in vision_agent/tools/tools.py

def siglip_classification(image: np.ndarray, labels: List[str]) -> Dict[str, Any]:
    """'siglip_classification' is a tool that can classify an image or a cropped detection given a list
    of input labels or tags. It returns the same list of the input labels along with
    their probability scores based on image content.

    Parameters:
        image (np.ndarray): The image to classify or tag
        labels (List[str]): The list of labels or tags that is associated with the image

    Returns:
        Dict[str, Any]: A dictionary containing the labels and scores. One dictionary
            contains a list of given labels and other a list of scores.

    Example
    -------
        >>> siglip_classification(image, ['dog', 'cat', 'bird'])
        {"labels": ["dog", "cat", "bird"], "scores": [0.68, 0.30, 0.02]},
    """

    if image.shape[0] < 1 or image.shape[1] < 1:
        return {"labels": [], "scores": []}

    image_file = numpy_to_bytes(image)

    files = [("image", image_file)]

    payload = {
        "model": "siglip",
        "labels": labels,
    }

    response: dict[str, Any] = send_inference_request(
        payload=payload,
        endpoint_name="classification",
        files=files,
        v2=True,
        metadata_payload={"function_name": "siglip_classification"},
    )

    return response

extract_frames_and_timestamps

extract_frames_and_timestamps(video_uri, fps=1)

'extract_frames_and_timestamps' extracts frames and timestamps from a video which can be a file path, url or youtube link, returns a list of dictionaries with keys "frame" and "timestamp" where "frame" is a numpy array and "timestamp" is the relative time in seconds where the frame was captured. The frame is a numpy array.

PARAMETER	DESCRIPTION
video_uri	The path to the video file, url or youtube link TYPE: Union[str, Path]
fps	The frame rate per second to extract the frames. Defaults to 1. TYPE: float DEFAULT: 1

RETURNS	DESCRIPTION
List[Dict[str, Union[ndarray, float]]]	List[Dict[str, Union[np.ndarray, float]]]: A list of dictionaries containing the extracted frame as a numpy array and the timestamp in seconds.

Example

>>> extract_frames("path/to/video.mp4")
[{"frame": np.ndarray, "timestamp": 0.0}, ...]

Source code in vision_agent/tools/tools.py

def extract_frames_and_timestamps(
    video_uri: Union[str, Path], fps: float = 1
) -> List[Dict[str, Union[np.ndarray, float]]]:
    """'extract_frames_and_timestamps' extracts frames and timestamps from a video
    which can be a file path, url or youtube link, returns a list of dictionaries
    with keys "frame" and "timestamp" where "frame" is a numpy array and "timestamp" is
    the relative time in seconds where the frame was captured. The frame is a numpy
    array.

    Parameters:
        video_uri (Union[str, Path]): The path to the video file, url or youtube link
        fps (float, optional): The frame rate per second to extract the frames. Defaults
            to 1.

    Returns:
        List[Dict[str, Union[np.ndarray, float]]]: A list of dictionaries containing the
            extracted frame as a numpy array and the timestamp in seconds.

    Example
    -------
        >>> extract_frames("path/to/video.mp4")
        [{"frame": np.ndarray, "timestamp": 0.0}, ...]
    """
    if isinstance(fps, str):
        # fps could be a string when it's passed in from a web endpoint deployment
        fps = float(fps)

    def reformat(
        frames_and_timestamps: List[Tuple[np.ndarray, float]],
    ) -> List[Dict[str, Union[np.ndarray, float]]]:
        return [
            {"frame": frame, "timestamp": timestamp}
            for frame, timestamp in frames_and_timestamps
        ]

    if str(video_uri).startswith(
        (
            "http://www.youtube.com/",
            "https://www.youtube.com/",
            "http://youtu.be/",
            "https://youtu.be/",
        )
    ):
        with tempfile.TemporaryDirectory() as temp_dir:
            yt = YouTube(str(video_uri))
            # Download the highest resolution video
            video = (
                yt.streams.filter(progressive=True, file_extension="mp4")
                .order_by("resolution")
                .desc()
                .first()
            )
            if not video:
                raise Exception("No suitable video stream found")
            video_file_path = video.download(output_path=temp_dir)

            return reformat(extract_frames_from_video(video_file_path, fps))
    elif str(video_uri).startswith(("http", "https")):
        _, image_suffix = os.path.splitext(video_uri)
        with tempfile.NamedTemporaryFile(delete=False, suffix=image_suffix) as tmp_file:
            # Download the video and save it to the temporary file
            with urllib.request.urlopen(str(video_uri)) as response:
                tmp_file.write(response.read())
            return reformat(extract_frames_from_video(tmp_file.name, fps))

    return reformat(extract_frames_from_video(str(video_uri), fps))

save_json

save_json(data, file_path)

'save_json' is a utility function that saves data as a JSON file. It is helpful for saving data that contains NumPy arrays which are not JSON serializable.

PARAMETER	DESCRIPTION
data	The data to save. TYPE: Any
file_path	The path to save the JSON file. TYPE: str

Example

>>> save_json(data, "path/to/file.json")

Source code in vision_agent/tools/tools.py

def save_json(data: Any, file_path: str) -> None:
    """'save_json' is a utility function that saves data as a JSON file. It is helpful
    for saving data that contains NumPy arrays which are not JSON serializable.

    Parameters:
        data (Any): The data to save.
        file_path (str): The path to save the JSON file.

    Example
    -------
        >>> save_json(data, "path/to/file.json")
    """

    class NumpyEncoder(json.JSONEncoder):
        def default(self, obj: Any):  # type: ignore
            if isinstance(obj, np.ndarray):
                return obj.tolist()
            elif isinstance(obj, np.bool_):
                return bool(obj)
            return json.JSONEncoder.default(self, obj)

    Path(file_path).parent.mkdir(parents=True, exist_ok=True)
    with open(file_path, "w") as f:
        json.dump(data, f, cls=NumpyEncoder)

load_image

load_image(image_path)

'load_image' is a utility function that loads an image from the given file path string or an URL.

PARAMETER	DESCRIPTION
image_path	The path or URL to the image. TYPE: str

RETURNS	DESCRIPTION
ndarray	np.ndarray: The image as a NumPy array.

Example

>>> load_image("path/to/image.jpg")

Source code in vision_agent/tools/tools.py

def load_image(image_path: str) -> np.ndarray:
    """'load_image' is a utility function that loads an image from the given file path string or an URL.

    Parameters:
        image_path (str): The path or URL to the image.

    Returns:
        np.ndarray: The image as a NumPy array.

    Example
    -------
        >>> load_image("path/to/image.jpg")
    """
    # NOTE: sometimes the generated code pass in a NumPy array
    if isinstance(image_path, np.ndarray):
        return image_path
    if image_path.startswith(("http", "https")):
        _, image_suffix = os.path.splitext(image_path)
        with tempfile.NamedTemporaryFile(delete=False, suffix=image_suffix) as tmp_file:
            # Download the image and save it to the temporary file
            with urllib.request.urlopen(image_path) as response:
                tmp_file.write(response.read())
            image_path = tmp_file.name
    image = Image.open(image_path).convert("RGB")
    return np.array(image)

save_image

save_image(image, file_path)

'save_image' is a utility function that saves an image to a file path.

PARAMETER	DESCRIPTION
image	The image to save. TYPE: ndarray
file_path	The path to save the image file. TYPE: str

Example

>>> save_image(image)

Source code in vision_agent/tools/tools.py

def save_image(image: np.ndarray, file_path: str) -> None:
    """'save_image' is a utility function that saves an image to a file path.

    Parameters:
        image (np.ndarray): The image to save.
        file_path (str): The path to save the image file.

    Example
    -------
        >>> save_image(image)
    """
    Path(file_path).parent.mkdir(parents=True, exist_ok=True)
    from IPython.display import display

    if not isinstance(image, np.ndarray) or (
        image.shape[0] == 0 and image.shape[1] == 0
    ):
        raise ValueError("The image is not a valid NumPy array with shape (H, W, C)")

    pil_image = Image.fromarray(image.astype(np.uint8)).convert("RGB")
    display(pil_image)
    pil_image.save(file_path)

save_video

save_video(frames, output_video_path=None, fps=1)

'save_video' is a utility function that saves a list of frames as a mp4 video file on disk.

PARAMETER	DESCRIPTION
frames	A list of frames to save. TYPE: list[ndarray]
output_video_path	The path to save the video file. If not provided, a temporary file will be created. TYPE: str DEFAULT: None
fps	The number of frames composes a second in the video. TYPE: float DEFAULT: 1

RETURNS	DESCRIPTION
str	The path to the saved video file. TYPE: str

Example

>>> save_video(frames)
"/tmp/tmpvideo123.mp4"

Source code in vision_agent/tools/tools.py

def save_video(
    frames: List[np.ndarray], output_video_path: Optional[str] = None, fps: float = 1
) -> str:
    """'save_video' is a utility function that saves a list of frames as a mp4 video file on disk.

    Parameters:
        frames (list[np.ndarray]): A list of frames to save.
        output_video_path (str): The path to save the video file. If not provided, a temporary file will be created.
        fps (float): The number of frames composes a second in the video.

    Returns:
        str: The path to the saved video file.

    Example
    -------
        >>> save_video(frames)
        "/tmp/tmpvideo123.mp4"
    """
    if isinstance(fps, str):
        # fps could be a string when it's passed in from a web endpoint deployment
        fps = float(fps)
    if fps <= 0:
        raise ValueError(f"fps must be greater than 0 got {fps}")

    if not isinstance(frames, list) or len(frames) == 0:
        raise ValueError("Frames must be a list of NumPy arrays")

    for frame in frames:
        if not isinstance(frame, np.ndarray) or (
            frame.shape[0] == 0 and frame.shape[1] == 0
        ):
            raise ValueError("A frame is not a valid NumPy array with shape (H, W, C)")

    if output_video_path is None:
        output_video_path = tempfile.NamedTemporaryFile(
            delete=False, suffix=".mp4"
        ).name
    else:
        Path(output_video_path).parent.mkdir(parents=True, exist_ok=True)

    output_video_path = video_writer(frames, fps, output_video_path)
    _save_video_to_result(output_video_path)
    return output_video_path

overlay_bounding_boxes

overlay_bounding_boxes(medias, bboxes)

'overlay_bounding_boxes' is a utility function that displays bounding boxes on an image.

PARAMETER	DESCRIPTION
medias	The image or frames to display the bounding boxes on. TYPE: Union[ndarray, List[ndarra]]
bboxes	A list of dictionaries or a list of list of dictionaries containing the bounding boxes. TYPE: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]

RETURNS	DESCRIPTION
Union[ndarray, List[ndarray]]	np.ndarray: The image with the bounding boxes, labels and scores displayed.

Example

>>> image_with_bboxes = overlay_bounding_boxes(
    image, [{'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]}],
)

Source code in vision_agent/tools/tools.py

def overlay_bounding_boxes(
    medias: Union[np.ndarray, List[np.ndarray]],
    bboxes: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]],
) -> Union[np.ndarray, List[np.ndarray]]:
    """'overlay_bounding_boxes' is a utility function that displays bounding boxes on
    an image.

    Parameters:
        medias (Union[np.ndarray, List[np.ndarra]]): The image or frames to display the
            bounding boxes on.
        bboxes (Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]): A list of
            dictionaries or a list of list of dictionaries containing the bounding
            boxes.

    Returns:
        np.ndarray: The image with the bounding boxes, labels and scores displayed.

    Example
    -------
        >>> image_with_bboxes = overlay_bounding_boxes(
            image, [{'score': 0.99, 'label': 'dinosaur', 'bbox': [0.1, 0.11, 0.35, 0.4]}],
        )
    """

    medias_int: List[np.ndarray] = (
        [medias] if isinstance(medias, np.ndarray) else medias
    )
    if len(bboxes) == 0:
        bbox_int: List[List[Dict[str, Any]]] = [[] for _ in medias_int]
    else:
        if isinstance(bboxes[0], dict):
            bbox_int = [cast(List[Dict[str, Any]], bboxes)]
        else:
            bbox_int = cast(List[List[Dict[str, Any]]], bboxes)

    labels = set([bb["label"] for b in bbox_int for bb in b])

    if len(labels) > len(COLORS):
        _LOGGER.warning(
            "Number of unique labels exceeds the number of available colors. Some labels may have the same color."
        )

    color = {label: COLORS[i % len(COLORS)] for i, label in enumerate(labels)}

    frame_out = []
    for i, frame in enumerate(medias_int):
        pil_image = Image.fromarray(frame.astype(np.uint8)).convert("RGB")

        bboxes = bbox_int[i]
        bboxes = sorted(bboxes, key=lambda x: x["label"], reverse=True)

        # if more than 50 boxes use small boxes to indicate objects else use regular boxes
        if len(bboxes) > 50:
            pil_image = _plot_counting(pil_image, bboxes, color)
        else:
            width, height = pil_image.size
            fontsize = max(12, int(min(width, height) / 40))
            draw = ImageDraw.Draw(pil_image)
            font = ImageFont.truetype(
                str(
                    resources.files("vision_agent.fonts").joinpath(
                        "default_font_ch_en.ttf"
                    )
                ),
                fontsize,
            )

            for elt in bboxes:
                label = elt["label"]
                box = elt["bbox"]
                scores = elt["score"]

                # denormalize the box if it is normalized
                box = denormalize_bbox(box, (height, width))
                draw.rectangle(box, outline=color[label], width=4)
                text = f"{label}: {scores:.2f}"
                text_box = draw.textbbox((box[0], box[1]), text=text, font=font)
                draw.rectangle(
                    (box[0], box[1], text_box[2], text_box[3]), fill=color[label]
                )
                draw.text((box[0], box[1]), text, fill="black", font=font)

        frame_out.append(np.array(pil_image))
    return_frame = frame_out[0] if len(frame_out) == 1 else frame_out

    if isinstance(return_frame, np.ndarray):
        from IPython.display import display

        display(Image.fromarray(return_frame))

    return return_frame  # type: ignore

overlay_segmentation_masks

overlay_segmentation_masks(
    medias,
    masks,
    draw_label=True,
    secondary_label_key="tracking_label",
)

'overlay_segmentation_masks' is a utility function that displays segmentation masks.

PARAMETER	DESCRIPTION
medias	The image or frames to display the masks on. TYPE: Union[ndarray, List[ndarray]]
masks	A list of dictionaries or a list of list of dictionaries containing the masks, labels and scores. TYPE: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]
draw_label	If True, the labels will be displayed on the image. TYPE: bool DEFAULT: True
secondary_label_key	The key to use for the secondary tracking label which is needed in videos to display tracking information. TYPE: str DEFAULT: 'tracking_label'

RETURNS	DESCRIPTION
Union[ndarray, List[ndarray]]	np.ndarray: The image with the masks displayed.

Example

>>> image_with_masks = overlay_segmentation_masks(
    image,
    [{
        'score': 0.99,
        'label': 'dinosaur',
        'mask': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
    }],
)

Source code in vision_agent/tools/tools.py

def overlay_segmentation_masks(
    medias: Union[np.ndarray, List[np.ndarray]],
    masks: Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]],
    draw_label: bool = True,
    secondary_label_key: str = "tracking_label",
) -> Union[np.ndarray, List[np.ndarray]]:
    """'overlay_segmentation_masks' is a utility function that displays segmentation
    masks.

    Parameters:
        medias (Union[np.ndarray, List[np.ndarray]]): The image or frames to display
            the masks on.
        masks (Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]): A list of
            dictionaries or a list of list of dictionaries containing the masks, labels
            and scores.
        draw_label (bool, optional): If True, the labels will be displayed on the image.
        secondary_label_key (str, optional): The key to use for the secondary
            tracking label which is needed in videos to display tracking information.

    Returns:
        np.ndarray: The image with the masks displayed.

    Example
    -------
        >>> image_with_masks = overlay_segmentation_masks(
            image,
            [{
                'score': 0.99,
                'label': 'dinosaur',
                'mask': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0]], dtype=uint8),
            }],
        )
    """
    if not masks:
        return medias

    medias_int: List[np.ndarray] = (
        [medias] if isinstance(medias, np.ndarray) else medias
    )
    masks_int = [masks] if isinstance(masks[0], dict) else masks
    masks_int = cast(List[List[Dict[str, Any]]], masks_int)

    labels = set()
    for mask_i in masks_int:
        for mask_j in mask_i:
            labels.add(mask_j["label"])
    color = {label: COLORS[i % len(COLORS)] for i, label in enumerate(labels)}

    width, height = Image.fromarray(medias_int[0]).size
    fontsize = max(12, int(min(width, height) / 40))
    font = ImageFont.truetype(
        str(resources.files("vision_agent.fonts").joinpath("default_font_ch_en.ttf")),
        fontsize,
    )

    frame_out = []
    for i, frame in enumerate(medias_int):
        pil_image = Image.fromarray(frame.astype(np.uint8)).convert("RGBA")
        for elt in masks_int[i]:
            mask = elt["mask"]
            label = elt["label"]
            tracking_lbl = elt.get(secondary_label_key, None)
            np_mask = np.zeros((pil_image.size[1], pil_image.size[0], 4))
            np_mask[mask > 0, :] = color[label] + (255 * 0.5,)
            mask_img = Image.fromarray(np_mask.astype(np.uint8))
            pil_image = Image.alpha_composite(pil_image, mask_img)

            if draw_label:
                draw = ImageDraw.Draw(pil_image)
                text = tracking_lbl if tracking_lbl else label
                text_box = draw.textbbox((0, 0), text=text, font=font)
                x, y = _get_text_coords_from_mask(
                    mask,
                    v_gap=(text_box[3] - text_box[1]) + 10,
                    h_gap=(text_box[2] - text_box[0]) // 2,
                )
                if x != 0 and y != 0:
                    text_box = draw.textbbox((x, y), text=text, font=font)
                    draw.rectangle((x, y, text_box[2], text_box[3]), fill=color[label])
                    draw.text((x, y), text, fill="black", font=font)
        frame_out.append(np.array(pil_image))
    return_frame = frame_out[0] if len(frame_out) == 1 else frame_out

    if isinstance(return_frame, np.ndarray):
        from IPython.display import display

        display(Image.fromarray(return_frame))

    return return_frame  # type: ignore

overlay_heat_map

overlay_heat_map(image, heat_map, alpha=0.8)

'overlay_heat_map' is a utility function that displays a heat map on an image.

PARAMETER	DESCRIPTION
image	The image to display the heat map on. TYPE: ndarray
heat_map	A dictionary containing the heat map under the key 'heat_map'. TYPE: Dict[str, Any]
alpha	The transparency of the overlay. Defaults to 0.8. TYPE: float DEFAULT: 0.8

RETURNS	DESCRIPTION
ndarray	np.ndarray: The image with the heat map displayed.

Example

>>> image_with_heat_map = overlay_heat_map(
    image,
    {
        'heat_map': array([[0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 0, 0, 0],
            ...,
            [0, 0, 0, ..., 0, 0, 0],
            [0, 0, 0, ..., 125, 125, 125]], dtype=uint8),
    },
)

Source code in vision_agent/tools/tools.py

def overlay_heat_map(
    image: np.ndarray, heat_map: Dict[str, Any], alpha: float = 0.8
) -> np.ndarray:
    """'overlay_heat_map' is a utility function that displays a heat map on an image.

    Parameters:
        image (np.ndarray): The image to display the heat map on.
        heat_map (Dict[str, Any]): A dictionary containing the heat map under the key
            'heat_map'.
        alpha (float, optional): The transparency of the overlay. Defaults to 0.8.

    Returns:
        np.ndarray: The image with the heat map displayed.

    Example
    -------
        >>> image_with_heat_map = overlay_heat_map(
            image,
            {
                'heat_map': array([[0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 0, 0, 0],
                    ...,
                    [0, 0, 0, ..., 0, 0, 0],
                    [0, 0, 0, ..., 125, 125, 125]], dtype=uint8),
            },
        )
    """
    pil_image = Image.fromarray(image.astype(np.uint8)).convert("RGB")

    if "heat_map" not in heat_map or len(heat_map["heat_map"]) == 0:
        return image

    pil_image = pil_image.convert("L")
    mask = Image.fromarray(heat_map["heat_map"])
    mask = mask.resize(pil_image.size)

    overlay = Image.new("RGBA", mask.size)
    odraw = ImageDraw.Draw(overlay)
    odraw.bitmap((0, 0), mask, fill=(255, 0, 0, round(alpha * 255)))
    combined = Image.alpha_composite(
        pil_image.convert("RGBA"), overlay.resize(pil_image.size)
    )
    return np.array(combined)

vision_agent.tools.tool_utils

ToolCallTrace

Bases: BaseModel

endpoint_url instance-attribute

endpoint_url

request instance-attribute

request

response instance-attribute

response

error instance-attribute

error

files instance-attribute

files

send_inference_request

send_inference_request(
    payload,
    endpoint_name,
    files=None,
    v2=False,
    metadata_payload=None,
    is_form=False,
)

Source code in vision_agent/tools/tool_utils.py

def send_inference_request(
    payload: Dict[str, Any],
    endpoint_name: str,
    files: Optional[List[Tuple[Any, ...]]] = None,
    v2: bool = False,
    metadata_payload: Optional[Dict[str, Any]] = None,
    is_form: bool = False,
) -> Any:
    # TODO: runtime_tag and function_name should be metadata_payload and not included
    # in the service payload
    if runtime_tag := os.environ.get("RUNTIME_TAG", ""):
        payload["runtime_tag"] = runtime_tag

    url = f"{_LND_API_URL_v2 if v2 else _LND_API_URL}/{endpoint_name}"
    if "TOOL_ENDPOINT_URL" in os.environ:
        url = os.environ["TOOL_ENDPOINT_URL"]

    headers = {"apikey": _LND_API_KEY}
    if "TOOL_ENDPOINT_AUTH" in os.environ:
        headers["Authorization"] = os.environ["TOOL_ENDPOINT_AUTH"]
        headers.pop("apikey")

    session = _create_requests_session(
        url=url,
        num_retry=3,
        headers=headers,
    )

    function_name = "unknown"
    if "function_name" in payload:
        function_name = payload["function_name"]
    elif metadata_payload is not None and "function_name" in metadata_payload:
        function_name = metadata_payload["function_name"]

    response = _call_post(url, payload, session, files, function_name, is_form)

    # TODO: consider making the response schema the same between below two sources
    return response if "TOOL_ENDPOINT_AUTH" in os.environ else response["data"]

send_task_inference_request

send_task_inference_request(
    payload,
    task_name,
    files=None,
    metadata=None,
    is_form=False,
)

Source code in vision_agent/tools/tool_utils.py

def send_task_inference_request(
    payload: Dict[str, Any],
    task_name: str,
    files: Optional[List[Tuple[Any, ...]]] = None,
    metadata: Optional[Dict[str, Any]] = None,
    is_form: bool = False,
) -> Any:
    url = f"{_LND_API_URL_v2}/{task_name}"
    headers = {"apikey": _LND_API_KEY}
    session = _create_requests_session(
        url=url,
        num_retry=3,
        headers=headers,
    )

    function_name = "unknown"
    if metadata is not None and "function_name" in metadata:
        function_name = metadata["function_name"]
    response = _call_post(url, payload, session, files, function_name, is_form)
    return response["data"]

get_tool_documentation

get_tool_documentation(funcs)

Source code in vision_agent/tools/tool_utils.py

def get_tool_documentation(funcs: List[Callable[..., Any]]) -> str:
    docstrings = ""
    for func in funcs:
        docstrings += f"{func.__name__}{inspect.signature(func)}:\n{func.__doc__}\n\n"

    return docstrings

get_tool_descriptions

get_tool_descriptions(funcs)

Source code in vision_agent/tools/tool_utils.py

def get_tool_descriptions(funcs: List[Callable[..., Any]]) -> str:
    descriptions = ""
    for func in funcs:
        description = func.__doc__
        if description is None:
            description = ""

        if "Parameters:" in description:
            description = (
                description[: description.find("Parameters:")]
                .replace("\n", " ")
                .strip()
            )

        description = " ".join(description.split())
        descriptions += f"- {func.__name__}{inspect.signature(func)}: {description}\n"
    return descriptions

get_tool_descriptions_by_names

get_tool_descriptions_by_names(
    tool_name, funcs, util_funcs
)

Source code in vision_agent/tools/tool_utils.py

def get_tool_descriptions_by_names(
    tool_name: Optional[List[str]],
    funcs: List[Callable[..., Any]],
    util_funcs: List[
        Callable[..., Any]
    ],  # util_funcs will always be added to the list of functions
) -> str:
    if tool_name is None:
        return get_tool_descriptions(funcs + util_funcs)

    invalid_names = [
        name for name in tool_name if name not in {func.__name__ for func in funcs}
    ]

    if invalid_names:
        raise ValueError(f"Invalid customized tool names: {', '.join(invalid_names)}")

    filtered_funcs = (
        funcs
        if not tool_name
        else [func for func in funcs if func.__name__ in tool_name]
    )
    return get_tool_descriptions(filtered_funcs + util_funcs)

get_tools_df

get_tools_df(funcs)

Source code in vision_agent/tools/tool_utils.py

def get_tools_df(funcs: List[Callable[..., Any]]) -> pd.DataFrame:
    data: Dict[str, List[str]] = {"desc": [], "doc": [], "name": []}

    for func in funcs:
        desc = func.__doc__
        if desc is None:
            desc = ""
        desc = desc[: desc.find("Parameters:")].replace("\n", " ").strip()
        desc = " ".join(desc.split())

        doc = f"{func.__name__}{inspect.signature(func)}:\n{func.__doc__}"
        data["desc"].append(desc)
        data["doc"].append(doc)
        data["name"].append(func.__name__)

    return pd.DataFrame(data)  # type: ignore

get_tools_info

get_tools_info(funcs)

Source code in vision_agent/tools/tool_utils.py

def get_tools_info(funcs: List[Callable[..., Any]]) -> Dict[str, str]:
    data: Dict[str, str] = {}

    for func in funcs:
        desc = func.__doc__
        if desc is None:
            desc = ""

        data[func.__name__] = f"{func.__name__}{inspect.signature(func)}:\n{desc}"

    return data

filter_bboxes_by_threshold

filter_bboxes_by_threshold(bboxes, threshold)

Source code in vision_agent/tools/tool_utils.py

def filter_bboxes_by_threshold(
    bboxes: BoundingBoxes, threshold: float
) -> BoundingBoxes:
    return list(filter(lambda bbox: bbox.score >= threshold, bboxes))

add_bboxes_from_masks

add_bboxes_from_masks(all_preds)

Source code in vision_agent/tools/tool_utils.py

def add_bboxes_from_masks(
    all_preds: List[List[Dict[str, Any]]],
) -> List[List[Dict[str, Any]]]:
    for frame_preds in all_preds:
        for preds in frame_preds:
            if np.sum(preds["mask"]) == 0:
                preds["bbox"] = []
            else:
                rows, cols = np.where(preds["mask"])
                bbox = [
                    float(np.min(cols)),
                    float(np.min(rows)),
                    float(np.max(cols)),
                    float(np.max(rows)),
                ]
                bbox = normalize_bbox(bbox, preds["mask"].shape)
                preds["bbox"] = bbox

    return all_preds

calculate_iou

calculate_iou(bbox1, bbox2)

Source code in vision_agent/tools/tool_utils.py

def calculate_iou(bbox1: List[float], bbox2: List[float]) -> float:
    x1, y1, x2, y2 = bbox1
    x3, y3, x4, y4 = bbox2

    x_overlap = max(0, min(x2, x4) - max(x1, x3))
    y_overlap = max(0, min(y2, y4) - max(y1, y3))
    intersection = x_overlap * y_overlap

    area1 = (x2 - x1) * (y2 - y1)
    area2 = (x4 - x3) * (y4 - y3)
    union = area1 + area2 - intersection

    return intersection / union if union > 0 else 0

single_nms

single_nms(preds, iou_threshold)

Source code in vision_agent/tools/tool_utils.py

def single_nms(
    preds: List[Dict[str, Any]], iou_threshold: float
) -> List[Dict[str, Any]]:
    for i in range(len(preds)):
        for j in range(i + 1, len(preds)):
            if calculate_iou(preds[i]["bbox"], preds[j]["bbox"]) > iou_threshold:
                if preds[i]["score"] > preds[j]["score"]:
                    preds[j]["score"] = 0
                else:
                    preds[i]["score"] = 0

    return [pred for pred in preds if pred["score"] > 0]

nms

nms(all_preds, iou_threshold)

Source code in vision_agent/tools/tool_utils.py

def nms(
    all_preds: List[List[Dict[str, Any]]], iou_threshold: float
) -> List[List[Dict[str, Any]]]:
    return_preds = []
    for frame_preds in all_preds:
        frame_preds = single_nms(frame_preds, iou_threshold)
        return_preds.append(frame_preds)

    return return_preds