dasda

SeleniumLinksGenerator.cs

using System.Collections.ObjectModel;
using System.Diagnostics;
using Common.Dtos;
using Common.Models;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Firefox;
using OpenQA.Selenium.Remote;
using OpenQA.Selenium.Support.UI;
using Selenium.Services;
using SeleniumExtras.WaitHelpers;

namespace Selenium;

public class SeleniumLinksGenerator : BackgroundService
{
    private IWebDriver? _driver;
    private readonly string _username;
    private readonly string _password;

    private readonly int _maxLinksPeerSession;

    // private readonly string _gridUrl = "http://seleniumgrid2.sportifier.com:4455/wd/hub";
    // private readonly string _gridUrl = "http://seleniumgrid1.sportifier.com:4455/wd/hub";
    private readonly string _dockerGrid = "http://chrome:4444/wd/hub";
    private readonly string _baseUrl = "https://www.bet365partners.com/app/#/LF/";
    private int _currentSessionLinks;
    private readonly IDictionary<string, string> _languageToLanguageCode;
    private readonly ILinksPoolRepository _repository;
    private readonly ILogger<SeleniumLinksGenerator> _logger;
    private readonly IConfiguration _configuration;

    public SeleniumLinksGenerator(IConfiguration configuration, ILinksPoolRepository repository,
                                  ILogger<SeleniumLinksGenerator> logger)
    {
        _configuration = configuration;
        _username = configuration["Bet365UserName"];
        _password = configuration["Bet365Password"];
        _maxLinksPeerSession = int.TryParse(configuration["MaxLinksPerSession"], out var amount) ? amount : 20;
        _languageToLanguageCode = configuration.GetSection("LanguageToLanguageCode")
                                               .Get<Dictionary<string, string>>()
                                               .ToDictionary(x => x.Key.ToLower(), x => x.Value,
                                                             StringComparer.OrdinalIgnoreCase);
        _repository = repository;
        _logger = logger;
    }

    private IWebDriver WebDriver
    {
        get
        {
            if (_driver is null || _currentSessionLinks > _maxLinksPeerSession)
            {
                InitNewDriver();
            }

            return _driver;
        }
    }

    private void InitNewDriver()
    {
        _driver?.Dispose();
        _currentSessionLinks = 0;
        var options = new ChromeOptions();
        options.AddArguments("--no-sandbox");    // Bypass OS security
        options.AddArguments("start-maximized"); // open Browser in maximized mode
        options.AddArguments("--headless");
        // options.AddArguments("--single-process");
        options.AddArguments("--disable-dev-shm-usage");                       // overcome limited resource problems
        // options.AddArguments("--incognito");                                   // incognito mode (private profile)
        // options.AddArguments("--disable-extensions");                          // disabling extensions
        // options.AddArguments("--disable-blink-features=AutomationControlled"); // disabling blink features");         
        // options.AddArguments("--disable-gpu");                                 // applicable to windows os only
        // options.AddArguments("--remote-debugging-port=9222");                  // Bypass OS security
        // options.AddArguments("disable-infobars");                              // disabling infobars
        
        _logger.LogInformation("grid {grid}", _dockerGrid);
        _driver = new RemoteWebDriver(new Uri(_dockerGrid), options);
        // _driver = new ChromeDriver(options);
        // _driver = new ChromeDriver();
        _driver.Manage().Timeouts().ImplicitWait = TimeSpan.FromSeconds(60);
        _driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(60);
        _driver.Manage().Window.Maximize();
        _driver.Navigate().GoToUrl(_baseUrl);
        Login();
    }

    private (bool, ReadOnlyCollection<IWebElement>) RetryingFind(By by, int attempts = 5)
    {
        _logger.LogInformation("RetryingFind: {By}", by);
        var result = false;
        var webElements = new ReadOnlyCollection<IWebElement>(new List<IWebElement>());
        for (var attemptNumber = 0; attemptNumber < attempts; attemptNumber++)
        {
            try
            {
                _logger.LogInformation("RetryingFind attempt number {AttemptNumber}", attemptNumber);
                webElements = WebDriver.FindElements(by);
                if (webElements?.Any() == true)
                {
                    result = true;
                    break;
                }
            }
            catch (Exception e)
            {
                _logger.LogInformation("Retrying find");
            }
        }

        _logger.LogInformation("RetryingFind result: {Result}", result);
        return (result, webElements);
    }

    private bool RetryingFindAndSendKeys(By by, string keys, bool withRefreshBetweenTries, int attempts = 5)
    {
        _logger.LogInformation("RetryingFindAndSendKeys: {By} with {Keys}", by, keys);
        var result = false;
        IWebElement? element = null;
        for (var attemptNumber = 0; attemptNumber < attempts; attemptNumber++)
        {
            try
            {
                if (attemptNumber != 0 && withRefreshBetweenTries)
                {
                    WebDriver.Navigate().Refresh();
                }

                _logger.LogInformation("RetryingFindAndSendKeys attempt number {AttemptNumber}", attemptNumber);
                element = WebDriver.FindElement(by);
                if (element is not null)
                {
                    result = true;
                    break;
                }
            }
            catch (Exception e)
            {
                _logger.LogInformation("Retrying RetryingFindAndSendKeys");
            }
        }

        if (result)
        {
            element?.Clear();
            element?.SendKeys(keys);
        }

        _logger.LogInformation("RetryingFindAndSendKeys result: {Result}", result);
        return result;
    }

    private bool RetryingFindClick(By by, bool withRefreshBetweenTries, int attempts = 5)
    {
        _logger.LogInformation("RetryingFindClick: {@By}", by);
        var result = false;
        for (var attemptNumber = 0; attemptNumber < attempts; attemptNumber++)
        {
            try
            {
                _logger.LogInformation("RetryingFindClick attempt number {AttemptNumber}", attemptNumber);
                var element = WebDriver.FindElement(by);
                if (element is not null)
                {
                    element.Click();
                    result = true;
                    break;
                }
            }
            catch (Exception e)
            {
                if (withRefreshBetweenTries)
                {
                    WebDriver.Navigate().Refresh();
                }

                _logger.LogInformation("Retrying findAndClick");
            }
        }

        _logger.LogInformation("RetryingFindClick result: {Result}", result);
        return result;
    }

    private void Login()
    {
        _logger.LogInformation("Preforming Login");
        var result = RetryingFindAndSendKeys(By.XPath("//input[@type='text']"), _username, true);
        if (!result)
        {
            throw new Exception("Could not find username field");
        }

        result = RetryingFindAndSendKeys(By.XPath("//input[@type='password']"), _password, true);
        if (!result)
        {
            throw new Exception("Could not find password field");
        }

        var click =
            RetryingFindClick(By.XPath("//button[@class='lf-LoginForm_Button svc-PartnersButton svc-PartnersButton-stacked ']"),
                              withRefreshBetweenTries: false);
        if (!click)
        {
            throw new Exception("Could not login");
        }

        // new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
        // .Until(ExpectedConditions
        // .ElementIsVisible(By.XPath("//div[@class='hm-HeaderLink hm-HeaderLink_CRLI ']")));
        WebDriver.Navigate().GoToUrl("https://www.bet365partners.com/app/#/CR/LI/");
    }

    private Bet365Link GenerateLink(string lang)
    {
        var name = DtoHelper.RandomString(16);
        var bet365Link = GenerateLink(name, lang);
        return bet365Link;
    }

    private Bet365Link GenerateLink(string linkName, string lang)
    {
        string bet365Code;
        _logger.LogInformation("URL: {Url}", WebDriver.Url);

        WebDriver.Navigate().GoToUrl("https://www.bet365partners.com/app/#/CR/LI/");
        if (WebDriver.Url != "https://www.bet365partners.com/app/#/CR/LI/")
        {
            InitNewDriver();
        }

        LanguageSelector(lang, withRefreshBetweenTries: true);
        RetryingFindAndSendKeys(By.XPath("//input[@class='cm-LinksAdminTextInput_Input ']"), linkName, true);
        RetryingFindClick(By.ClassName("cm-LinksAdmin_InputLabel"), withRefreshBetweenTries: false);
        WebDriver.Navigate().Refresh();
        LanguageSelector(lang, withRefreshBetweenTries: true);
        var (result, elements) = GetLinkCode(linkName, lang);
        if (result)
        {
            var element = elements.FirstOrDefault(x => x.Text.Trim() == linkName);

            if (element != null)
            {
                var (result2, parentElement) = element.RetryingFind(By.XPath("../../.."));
                if (result2 && parentElement is not null)
                {
                    bet365Code = parentElement.GetAttribute("row_id");
                }
                else
                {
                    throw new Exception("Can't find parent element hence could not get link code");
                }
            }
            else
            {
                _logger.LogError("Link not found");
                throw new Exception($"Link {linkName} not found");
            }
        }
        else
        {
            bet365Code = GetLinkById(linkName, lang);
            if (string.IsNullOrWhiteSpace(bet365Code))
            {
                throw new Exception($"Link {linkName} not found");
            }
        }

        _currentSessionLinks++;
        return new Bet365Link { Name = linkName, Code = bet365Code, Language = lang };
    }

    private (bool, ReadOnlyCollection<IWebElement>) GetLinkCode(string linkName, string lang)
    {
        try
        {
            var elements = new ReadOnlyCollection<IWebElement>(new List<IWebElement>());
            new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                .Until(ExpectedConditions.ElementIsVisible(By.XPath("//div[@class='cm-LinksGrid ']")));
            new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                .Until(ExpectedConditions.ElementIsVisible(By.XPath("//div[@class='cm-LinksRow ']")));
            for (var i = 0; i < 5; i++)
            {
                RetryingFindAndSendKeys(By.ClassName("cm-PageHeaderSearch_SearchBoxInput"),
                                        linkName,
                                        withRefreshBetweenTries: false,
                                        attempts: 1);
                (var result, elements) =
                    RetryingFind(By.XPath("//span[@class='cm-LinksRow_ColumnLabel-highlight']"), attempts: 1);
                if (result)
                {
                    return (result, elements);
                }

                WebDriver.Navigate().Refresh();
                LanguageSelector(lang, false);
                new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                    .Until(ExpectedConditions.ElementIsVisible(By.XPath("//div[@class='cm-LinksGrid ']")));
                new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                    .Until(ExpectedConditions.ElementIsVisible(By.XPath("//div[@class='cm-LinksRow ']")));
            }
        }
        catch (Exception)
        {
            return (false, new ReadOnlyCollection<IWebElement>(new List<IWebElement>()));
        }

        return (false, new ReadOnlyCollection<IWebElement>(new List<IWebElement>()));
    }

    private void LanguageSelector(string lang, bool withRefreshBetweenTries)
    {
        var (result, elements) = RetryingFind(By.XPath("//*[@class='cm-LinksAdminDropDown_Select ']"));
        if (result)
        {
            var langDdl = elements.First();
            new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                .Until(ExpectedConditions.ElementIsVisible(By.ClassName("cm-LinksAdminDropDown_Placeholder")));
            var currLang = langDdl.FindElement(By.ClassName("cm-LinksAdminDropDown_Placeholder"))?.Text?.Trim();

            if (currLang != lang)
            {
                var langId = _languageToLanguageCode[lang];
                langDdl.Click();
                new WebDriverWait(WebDriver, TimeSpan.FromSeconds(60))
                    .Until(ExpectedConditions.ElementToBeClickable(By.XPath($"//div[@data-id='{langId}']")));

                var retryingFindClickResult = RetryingFindClick(By.XPath($"//div[@data-id='{langId}']"), withRefreshBetweenTries);
                if (!retryingFindClickResult)
                {
                    _logger.LogError("Can't find language");
                    throw new Exception("Can't find language");
                }
            }
        }
    }

    private string GetLinkById(string linkName, string lang)
    {
        var response = string.Empty;

        WebDriver.Navigate().Refresh();
        WebDriver.Navigate().GoToUrl("https://www.bet365partners.com/app/#/CR/LI/");
        LanguageSelector(lang, withRefreshBetweenTries: true);
        WebDriver.FindElement(By.ClassName("cm-PageHeaderSearch_SearchBoxInput")).SendKeys(linkName);

        var waitForSearchResults = new WebDriverWait(WebDriver, TimeSpan.FromSeconds(5));
        waitForSearchResults.Until(ExpectedConditions.ElementExists(By.ClassName("cm-LinksRow_ColumnLabel-highlight")));

        var searchResults = WebDriver.FindElements(By.ClassName("cm-LinksRow_ColumnLabel-highlight"));

        var element = searchResults.FirstOrDefault(x => x.Text.Trim() == linkName);

        if (element != null)
        {
            var parentElement = element.FindElement(By.XPath("../../.."));
            response = parentElement.GetAttribute("row_id");
        }

        return response;
    }

    private async Task StartGenerating()
    {
        var linksPollConfig = _configuration.GetSection("LinksPool")
                                            .Get<Dictionary<string, int>>()
                                            .ToDictionary(x => x.Key, x => x.Value,
                                                          StringComparer.OrdinalIgnoreCase);
        //iterate the config and create a pool of links for each language

        for (var index = 0;; index = (index + 1) % linksPollConfig.Count)
        {
            var keyValuePair = linksPollConfig.ElementAt(index);
            var currentAmountOfLinks = await _repository.GetLinksCountByLanguage(keyValuePair.Key);
            var amountOfLinksToGenerate = keyValuePair.Value - currentAmountOfLinks;
            _logger.LogInformation("Creating pool for {Country} with {Amount} links", keyValuePair.Key, keyValuePair.Value);
            _logger.LogInformation("Current amount of {Country} links: {CurrentAmount}", keyValuePair.Key,
                                   currentAmountOfLinks);
            _logger.LogInformation("Amount of links to generate: {AmountToGenerate}", amountOfLinksToGenerate);

            try
            {
                for (var i = 0; i < amountOfLinksToGenerate; i++)
                {
                    var bet365Link = GenerateLink(keyValuePair.Key);
                    _logger.LogInformation("=========================================================================================");
                    _logger.LogInformation("Created link: {Link}", bet365Link.Code);
                    _logger.LogInformation("=========================================================================================");
                    await _repository.AddLinkAsync(bet365Link);
                }
            }
            catch (Exception e)
            {
                _logger.LogError(e, "Error while creating pool for {Country}", keyValuePair.Key);
                InitNewDriver();
            }
        }
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogInformation("Starting link pool generator");
        var process = Process.GetCurrentProcess(); // Or whatever method you are using
        string? fullPath = process.MainModule?.FileName;
        _logger.LogInformation("Full path: {FullPath}", fullPath);
        await StartGenerating();
    }
}