Click Element by Description

interface ClickByDescriptionRequest {
  taskDescription: string;
  imageBase64?: string;
  mechanism?: string;
}

interface ClickByDescriptionResponse {
  success: boolean;
  message: string;
  timestamp: string;
}

/**
 * Use AI vision to find and click a UI element based on its description
 * 
 * @param apiUrl - Base URL of the Tools Server API
 * @param apiKey - API key for authentication
 * @param elementDescription - Description of the UI element to click
 * @returns Promise with the response containing success status and coordinates
 */
async function clickElementByDescription(
  apiUrl: string,
  apiKey: string,
  elementDescription: string
): Promise {
  const endpoint = `${apiUrl}/tools-api/mouse/click-by-description`;
  
  // Prepare the request payload
  const payload: ClickByDescriptionRequest = {
    taskDescription: elementDescription
  };
  
  // Make the API request
  const response = await fetch(endpoint, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': `Bearer ${apiKey}`
    },
    body: JSON.stringify(payload)
  });
  
  // Parse and return the response
  return await response.json();
}

// Example usage
async function example() {
  const API_URL = 'http://localhost:8080';
  const API_KEY = 'your_api_key_here';
  
  try {
    // Example: Find and click the File menu
    const result = await clickElementByDescription(API_URL, API_KEY, 'the File menu');
    
    console.log(`Success: ${result.success}`);
    console.log(`Message: ${result.message}`);
    console.log(`Timestamp: ${result.timestamp}`);
  } catch (error) {
    console.error('API request failed:', error);
  }
}

using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

namespace SmoothOperatorClient
{
    public class ToolsServerClient
    {
        private readonly HttpClient _httpClient;
        private readonly string _apiKey;
        
        public ToolsServerClient(string apiUrl, string apiKey)
        {
            _httpClient = new HttpClient
            {
                BaseAddress = new Uri(apiUrl)
            };
            _apiKey = apiKey;
            _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", _apiKey);
        }
        
        /// 

        /// Use AI vision to find and click a UI element based on its description
        /// 
        /// Description of the UI element to click
        /// Response containing success status and click coordinates
        public async Task ClickElementByDescriptionAsync(string elementDescription)
        {
            var request = new ClickByDescriptionRequest
            {
                TaskDescription = elementDescription
            };
            
            var content = new StringContent(
                JsonSerializer.Serialize(request), 
                Encoding.UTF8, 
                "application/json");
                
            var response = await _httpClient.PostAsync("/tools-api/mouse/click-by-description", content);
            response.EnsureSuccessStatusCode();
            
            var responseBody = await response.Content.ReadAsStringAsync();
            return JsonSerializer.Deserialize(responseBody);
        }
    }
    
    public class ClickByDescriptionRequest
    {
        public string TaskDescription { get; set; }
        public string ImageBase64 { get; set; }
        public string Mechanism { get; set; }
    }
    
    public class ClickByDescriptionResponse
    {
        public bool Success { get; set; }
        public string Message { get; set; }
        public DateTime Timestamp { get; set; }
    }
    
    // Example usage
    class Program
    {
        static async Task Main(string[] args)
        {
            var client = new ToolsServerClient("http://localhost:8080", "your_api_key_here");
            
            try
            {
                // Example: Find and click the File menu
                var result = await client.ClickElementByDescriptionAsync("the File menu");
                
                Console.WriteLine($"Success: {result.Success}");
                Console.WriteLine($"Message: {result.Message}");
                Console.WriteLine($"Timestamp: {result.Timestamp}");
            }
            catch (Exception ex)
            {
                Console.WriteLine($"API request failed: {ex.Message}");
            }
        }
    }
}