Drag by Description

import axios from 'axios';

/**
 * Performs a drag operation from one element to another using AI vision
 * 
 * @param startElementDescription - Description of the UI element to drag from
 * @param endElementDescription - Description of the destination element or location
 * @param imageBase64 - Optional base64-encoded screenshot image
 * @param mechanism - Optional mechanism to use for vision detection
 * @returns Promise with the response data
 */
async function dragByDescription(
    startElementDescription: string,
    endElementDescription: string,
    imageBase64?: string,
    mechanism?: string
): Promise<{
    success: boolean;
    message: string;
    timestamp: string;
}> {
    // Base URL of the Tools Server
    const baseUrl = 'http://localhost:54321';
    
    // Endpoint
    const endpoint = '/tools-api/mouse/drag-by-description';
    
    // Request headers
    const headers = {
        'Content-Type': 'application/json',
        'Authorization': 'Bearer YOUR_API_KEY'
    };
    
    // Request payload
    const payload = {
        startElementDescription,
        endElementDescription,
        imageBase64, // If undefined, server will take a screenshot
        mechanism
    };
    
    try {
        // Send the request
        const response = await axios.post(
            baseUrl + endpoint,
            payload,
            { headers }
        );
        
        // Return the response data
        return response.data;
    } catch (error) {
        // Handle any errors
        console.error('Error performing drag operation:', error);
        throw error;
    }
}

// Example usage
async function example() {
    try {
        const result = await dragByDescription(
            'the file icon',
            'the folder icon',
            undefined,
            'screengrasp2'
        );
        
        console.log(`Success: ${result.success}`);
        console.log(`Message: ${result.message}`);
        console.log(`Timestamp: ${result.timestamp}`);
    } catch (error) {
        console.error('Failed to perform drag operation:', error);
    }
}

example();

using System;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Threading.Tasks;

public class ToolsServerClient
{
    private readonly HttpClient _httpClient;
    private readonly string _baseUrl;
    private readonly string _apiKey;
    
    public ToolsServerClient(string baseUrl = "http://localhost:54321", string apiKey = "YOUR_API_KEY")
    {
        _baseUrl = baseUrl;
        _apiKey = apiKey;
        
        _httpClient = new HttpClient();
        _httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {_apiKey}");
    }
    
    /// 

    /// Performs a drag operation from one element to another using AI vision
    /// 
    /// Description of the UI element to drag from
    /// Description of the destination element or location
    /// Optional base64-encoded screenshot image
    /// Optional mechanism to use for vision detection
    /// Action response indicating success or failure
    public async Task DragByDescriptionAsync(
        string startElementDescription,
        string endElementDescription,
        string imageBase64 = null,
        string mechanism = null)
    {
        // Endpoint
        var endpoint = "/tools-api/mouse/drag-by-description";
        
        // Create request object
        var request = new DragByDescriptionRequest
        {
            StartElementDescription = startElementDescription,
            EndElementDescription = endElementDescription,
            ImageBase64 = imageBase64, // If null, server will take a screenshot
            Mechanism = mechanism
        };
        
        // Serialize request to JSON
        var jsonContent = JsonSerializer.Serialize(request);
        var content = new StringContent(jsonContent, Encoding.UTF8, "application/json");
        
        // Send the request
        var response = await _httpClient.PostAsync(_baseUrl + endpoint, content);
        
        // Ensure success status code
        response.EnsureSuccessStatusCode();
        
        // Deserialize the response
        var jsonResponse = await response.Content.ReadAsStringAsync();
        return JsonSerializer.Deserialize(jsonResponse);
    }
}

// Request and response models
public class DragByDescriptionRequest
{
    public string StartElementDescription { get; set; }
    public string EndElementDescription { get; set; }
    public string ImageBase64 { get; set; }
    public string Mechanism { get; set; }
}

public class ActionResponse
{
    public bool Success { get; set; }
    public string Message { get; set; }
    public DateTime Timestamp { get; set; }
}

// Example usage
class Program
{
    static async Task Main()
    {
        var client = new ToolsServerClient();
        
        try
        {
            var result = await client.DragByDescriptionAsync(
                "the file icon",
                "the folder icon",
                mechanism: "screengrasp2"
            );
            
            Console.WriteLine($"Success: {result.Success}");
            Console.WriteLine($"Message: {result.Message}");
            Console.WriteLine($"Timestamp: {result.Timestamp}");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"Error: {ex.Message}");
        }
    }
}