Find UI Element

interface FindUiElementRequest {
  taskDescription: string;
  imageBase64?: string;
  mechanism?: string;
}

interface FindUiElementResponse {
  success: boolean;
  message: string;
  timestamp: string;
  x: number | null;
  y: number | null;
  status: string;
  imageBase64: string;
}

/**
 * Find UI elements on the screen using a natural language description.
 * 
 * @param description - Natural language description of the UI element to find
 * @param options - Additional options for the search
 * @returns A promise that resolves to the find UI element response
 */
async function findUiElement(
  description: string,
  options: {
    imageBase64?: string;
    mechanism?: string;
  } = {}
): Promise {
  const url = 'http://localhost:5000/tools-api/screenshot/find-ui-element';
  
  const headers: Record = {
    'Content-Type': 'application/json'
  };
  
  const request: FindUiElementRequest = {
    taskDescription: description,
    ...options
  };
  
  const response = await fetch(url, {
    method: 'POST',
    headers,
    body: JSON.stringify(request)
  });
  
  return await response.json();
}

/**
 * Generate HTML to visualize the elements found by drawing rectangles on the screenshot.
 * 
 * @param response - The response from the findUiElement function
 * @returns HTML string with the visualization
 */
function generateVisualization(response: FindUiElementResponse): string {
  if (!response.success || !response.imageBase64) {
    return '
No screenshot in response or operation failed
';
  }
  
  const containerStyle = 'position: relative; max-width: 100%; overflow: hidden;';
  
  let html = `
`;
  
  // Add the screenshot as background
  html += ``;
  
  // Add rectangles for each element
  if (response.x !== null && response.y !== null) {
    const rectStyle = 'position: absolute; left: ' + response.x + 'px; top: ' + response.y + 
                      'px; width: 10px; height: 10px; border: 2px solid red; background-color: rgba(255, 0, 0, 0.3);';
    
    html += `
`;
  }
  
  html += '
';
  return html;
}

// Example usage
(async () => {
  try {
    // Find buttons on the screen
    const result = await findUiElement(
      'submit button',
      {
        mechanism: 'screengrasp2-medium'
      }
    );
    
    // Print basic information
    console.log(`Success: ${result.success}`);
    console.log(`Message: ${result.message}`);
    console.log(`Timestamp: ${result.timestamp}`);
    console.log(`X: ${result.x}`);
    console.log(`Y: ${result.y}`);
    console.log(`Status: ${result.status}`);
    
    // Create visualization
    if (result.success && result.imageBase64) {
      const visualization = generateVisualization(result);
      // In a browser environment, you could add this to the DOM:
      // document.getElementById('visualization').innerHTML = visualization;
      console.log('Visualization HTML generated');
    }
  } catch (error) {
    console.error('Error finding UI element:', error);
  }
})();
                    

using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

public class FindUiElementRequest
{
    public string TaskDescription { get; set; }
    public string ImageBase64 { get; set; }
    public string Mechanism { get; set; }
}

public class FindUiElementResponse
{
    public bool Success { get; set; }
    public string Message { get; set; }
    public string Timestamp { get; set; }
    public int? X { get; set; }
    public int? Y { get; set; }
    public string Status { get; set; }
    public string ImageBase64 { get; set; }
}

public class ToolsServerClient
{
    private readonly HttpClient _httpClient;
    
    public ToolsServerClient()
    {
        _httpClient = new HttpClient();
        _httpClient.BaseAddress = new Uri("http://localhost:5000/");
    }
    
    /// 

    /// Find UI elements on the screen using a natural language description.
    /// 
    /// Natural language description of the UI element to find
    /// Base64 encoded screenshot image (if not provided, a screenshot will be taken)
    /// Vision mechanism to use (screengrasp2, screengrasp2-low, screengrasp2-medium, screengrasp2-high, llabs, anthropic-computer-use, openai-computer-use, qwen25-vl-72b)
    /// Response containing the found UI elements
    public async Task FindUiElementAsync(
        string description,
        string imageBase64 = null,
        string mechanism = "screengrasp2")
    {
        var request = new FindUiElementRequest
        {
            TaskDescription = description,
            ImageBase64 = imageBase64,
            Mechanism = mechanism
        };
        
        var json = JsonSerializer.Serialize(request);
        var content = new StringContent(json, Encoding.UTF8, "application/json");
        
        var response = await _httpClient.PostAsync("tools-api/screenshot/find-ui-element", content);
        response.EnsureSuccessStatusCode();
        
        var responseBody = await response.Content.ReadAsStringAsync();
        return JsonSerializer.Deserialize(responseBody, new JsonSerializerOptions
        {
            PropertyNameCaseInsensitive = true
        });
    }
    
    /// 

    /// Save the screenshot from the response to a file and generate an image with rectangles
    /// around the found UI elements.
    /// 
    /// The response from FindUiElementAsync
    /// Path to save the visualization image
    /// Path to the saved image
    public string SaveVisualization(FindUiElementResponse response, string outputPath)
    {
        if (!response.Success || string.IsNullOrEmpty(response.ImageBase64))
        {
            throw new InvalidOperationException("No screenshot in response or operation failed");
        }
        
        // Convert base64 to image
        var imageBytes = Convert.FromBase64String(response.ImageBase64);
        using var ms = new MemoryStream(imageBytes);
        using var originalImage = Image.FromStream(ms);
        
        // Create a copy to draw on
        using var image = new Bitmap(originalImage);
        using var graphics = Graphics.FromImage(image);
        
        // Draw rectangles for each element
        using var pen = new Pen(Color.Red, 2);
        
        if (response.X.HasValue && response.Y.HasValue)
        {
            // Draw rectangle
            graphics.DrawRectangle(pen, response.X.Value, response.Y.Value, 10, 10);
        }
        
        // Save the image
        image.Save(outputPath);
        return outputPath;
    }
}

// Example usage
class Program
{
    static async Task Main(string[] args)
    {
        var client = new ToolsServerClient();
        
        try
        {
            // Find buttons on the screen
            var result = await client.FindUiElementAsync(
                description: "login button",
                mechanism: "screengrasp2-medium"
            );
            
            // Print basic information
            Console.WriteLine($"Success: {result.Success}");
            Console.WriteLine($"Message: {result.Message}");
            Console.WriteLine($"Timestamp: {result.Timestamp}");
            Console.WriteLine($"X: {result.X}");
            Console.WriteLine($"Y: {result.Y}");
            Console.WriteLine($"Status: {result.Status}");
            
            // Save visualization
            if (result.Success && !string.IsNullOrEmpty(result.ImageBase64))
            {
                var outputPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop), "ui_elements.png");
                client.SaveVisualization(result, outputPath);
                Console.WriteLine($"Visualization saved to: {outputPath}");
            }
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error: {ex.Message}");
        }
    }
}