using Crawler.

Utilites;
using Crawler.Extensions.SeleniumExtensions;
using OpenQA.Selenium;
using OpenQA.Selenium.Interactions;

namespace JobsTrackerCrawler
{
class Program
{

static async Task Main(string[] args)

{
bool IsProgramCrash = false;
int retryCount = 0;
int maxRetries = 10;
int maxAttempts = 10;
int attempts = 0;

while (retryCount < maxRetries)

{
try
{
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string folderPath = Path.Combine(documentsPath, "Frase.io");
string downloadsFolder =
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
"Downloads");
string successLoggerPath = Path.Combine(folderPath,
"SuccessLog.txt");
string countryAndLanuagePath = Path.Combine(folderPath,
"CountryAndLanguage.txt");
var (country, language) =
GetCountryAndLanguage(countryAndLanuagePath);
var initialFiles =
Directory.GetFiles(downloadsFolder).ToHashSet();

string documentsFolder =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string baseFolder = Path.Combine(documentsFolder, "Frase.io",
"GeneratedContent");

UIConstants.SiteUrl = "https://app.frase.io/app/documents";
SeleniumBrowserMethods.InitializeAndSetupBrowser("ProfileOne");
SeleniumObjectsMethods.PutWaitInThread(2);

if (SeleniumBrowserMethods.GetCurrentUrl() != null &&

SeleniumBrowserMethods.GetCurrentUrl().Contains("login"))
{
SeleniumObjectsMethods.PutWaitInThread(5);
Login();
SeleniumObjectsMethods.PutWaitInThread(5);
}

string[] topics = GetTopicList();

foreach (var topic in topics)
{
if (!CheckInternetConnection())
 {
Console.WriteLine("Internet connection lost.
Retrying...");
Thread.Sleep(5000);
continue;
}
else
{
SeleniumBrowserMethods.Navigate(UIConstants.SiteUrl,
true);
}

CreateNewDocumentPopUp();
ChangeCountryAndLanguage(country, language);

SeleniumObjectsMethods.PutWaitInThread(5);
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[placeholder=\"Enter search query\"]", topic, 250);
SeleniumSetMethods.Click(PropertyType.XPath,
"//button[span[text()='Start']]", 250);

while (attempts < maxAttempts && !

SeleniumObjectsMethods.IsElementVisible(PropertyType.XPath, "//button[@role='tab'
and contains(@aria-controls, 'radix') and contains(text(), 'Outline')]"))
{
SeleniumObjectsMethods.PutWaitInThread(3); // Wait for
3 seconds
attempts++;

if (attempts >= maxAttempts)

{
Console.WriteLine("Max attempts reached: Element
not found.");
break; // Stop the loop after 10 tries
}
}

SeleniumObjectsMethods.PutWaitInThread(5);
SeleniumBrowserMethods.RefreshPage();
SeleniumObjectsMethods.PutWaitInThread(2);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[aria-controls*=\"-content-outline\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[class*=\"backdrop-blur-lg bg-emerald-200/30 dark:bg-emerald-800/30\"]",
250);

SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[id=\"optimize-tab\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[data-orientation=\"horizontal\"][data-state=\"active\"] button[aria-
haspopup=\"menu\"][aria-expanded=\"false\"][data-state=\"closed\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"a[download=\"topics.csv\"]", 250);

WaitForNewFiles(downloadsFolder, initialFiles, 600, topic);

SeleniumObjectsMethods.PutWaitInThread(5);

string topicFolder = Path.Combine(baseFolder, topic);

string latestCsvFile = GetLatestFile(downloadsFolder,
"*.csv");

if (!Directory.Exists(topicFolder))
{
Directory.CreateDirectory(topicFolder);
}

if (!string.IsNullOrEmpty(latestCsvFile))
{
string newCsvFilePath = Path.Combine(topicFolder, topic
+ ".csv");

if (File.Exists(newCsvFilePath))
{
File.Delete(newCsvFilePath);
Console.WriteLine($"Existing CSV file deleted:
{newCsvFilePath}");
}
SeleniumObjectsMethods.PutWaitInThread(4);
File.Move(latestCsvFile, newCsvFilePath);
}
else
{
Console.WriteLine("No CSV file found in Downloads.");
}

File.AppendAllText(successLoggerPath, $"'{topic}' has been

successfully generated by AI.{Environment.NewLine}");
Console.WriteLine($"The topic '{topic}' has been
successfully generated by AI.");

RemoveTopicFromFile(topic);
}

// If execution reaches here, reset retryCount

retryCount = 0;
break;
}
catch (Exception ex)
{
retryCount++;
Console.WriteLine($"Attempt {retryCount} failed:
{ex.Message}");
LogError("An unknown error occurred in the Scraper: " +
ex.Message + "\nStackTrace: " + ex.StackTrace);

if (retryCount >= maxRetries)

{
Console.WriteLine("Maximum retry attempts reached. Exiting
program.");
break;
}

Console.WriteLine("Retrying after 10 seconds...");

Thread.Sleep(10000); // Wait before retrying
}
finally
{
SeleniumBrowserMethods.DisposeBrowser();
 }
}
}

public static void RemoveTopicFromFile(string completedTopic)

{
try
{
string userDocumentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string topicListPath = Path.Combine(userDocumentsPath, "Frase.io",
"TopicList.txt");

if (File.Exists(topicListPath))
{
var topics = File.ReadAllLines(topicListPath)
.Where(line => !
string.IsNullOrWhiteSpace(line) && line.Trim() != completedTopic)
.ToList();

File.WriteAllLines(topicListPath, topics);

Console.WriteLine($"Removed completed topic:

{completedTopic}");
}
}
catch (Exception ex)
{
Console.WriteLine($"Error removing topic: {ex.Message}");
}
}

public static bool CheckInternetConnection()

{
try
{
using (var client = new System.Net.WebClient())
using (client.OpenRead("http://www.google.com"))
{
return true;
}
}
catch
{
return false;
}
}
static (string Email, string Password) Credentials()
{
// Get the Documents folder path dynamically
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string folderPath = Path.Combine(documentsPath, "Frase.io");
string filePath = Path.Combine(folderPath, "Credentials.txt");

try
{
// Check if the file exists
 if (File.Exists(filePath))
{
// Read all lines from the file
string[] lines = File.ReadAllLines(filePath);

// Ensure the file has at least two lines

if (lines.Length >= 2 && !string.IsNullOrWhiteSpace(lines[0])
&& !string.IsNullOrWhiteSpace(lines[1]))
{
return (lines[0].Trim(), lines[1].Trim()); // Return email
and password
}
else
{
Console.WriteLine("Error: The file does not contain a valid
email and password.");
}
}
else
{
Console.WriteLine("Error: Credentials file not found.");
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
}

return (string.Empty, string.Empty); // Return empty values if an error

occurs
}

static void Login()

{
SeleniumObjectsMethods.PutWaitInThread(3);
var (email, password) = Credentials();
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[type='email']", email, 250);
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[type='password'", password, 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[type=\"submit\"]", 250);
SeleniumObjectsMethods.PutWaitInThread(1);
}

static void CreateNewDocumentPopUp()

{
SeleniumObjectsMethods.PutWaitInThread(5);
//click on new document button
SeleniumSetMethods.Click(PropertyType.XPath,
"(//button[.//span[text()='New document']])[1]", 250);
SeleniumSetMethods.Click(PropertyType.XPath, "//span[text()='Blank
Document']/ancestor::div[contains(@class, 'transition-all')]", 250);
SeleniumSetMethods.Click(PropertyType.XPath,
"//button[span[text()='Continue']]", 250);
}

static void ChangeCountryAndLanguage(string country, string language)

{
 SeleniumObjectsMethods.PutWaitInThread(3);
if (!string.IsNullOrEmpty(country) && !string.IsNullOrEmpty(language))
{
SeleniumSetMethods.Click(PropertyType.XPath,
"//input[@placeholder='Enter search query']/following-sibling::div/button[2]", 60);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div[class^=\"space-y-2\"]:nth-child(2) button[aria-
haspopup=\"dialog\"]:nth-child(2)");
var countriesNames =
SeleniumGetMethods.GetWebElements(PropertyType.CssSelector, "div[role='group'] >
div");
foreach (var countriesName in countriesNames)
{
if
(SeleniumGetMethods.GetInnerTextFromHTML(countriesName).ToLower().Contains(country.
ToLower()))
{
SeleniumSetMethods.Click(countriesName);
Actions actions = new Actions(PropertiesCollection.Driver);
actions.SendKeys(Keys.Escape).Perform();

break;
}
}

SeleniumObjectsMethods.PutWaitInThread(2);

SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div[class^=\"space-y-2\"]:nth-child(3) button[aria-
haspopup=\"dialog\"]:nth-child(2)", 60);

var languageNames =
SeleniumGetMethods.GetWebElements(PropertyType.CssSelector, "div[role='group'] >
div");
foreach (var name in languageNames)
{
if
(SeleniumGetMethods.GetInnerTextFromHTML(name).ToLower().Contains(language.ToLower(
)))
{
SeleniumSetMethods.Click(name);
Actions actions = new Actions(PropertiesCollection.Driver);
actions.SendKeys(Keys.Escape).Perform();
break;
}
}

SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div:nth-child(4) > button:nth-child(2)");
}

static void LogError(string message)

{
// Get the Documents folder path dynamically
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
 // Define the log file path dynamically
string logFilePath = Path.Combine(documentsPath, "Frase.io",
"ErrorLog.txt");

// Ensure the directory exists

Directory.CreateDirectory(Path.GetDirectoryName(logFilePath));

// Write the error message to the log file

File.AppendAllText(logFilePath, $"{DateTime.Now}: {message}
{Environment.NewLine}");

Console.WriteLine("Error logged successfully.");

}

public static string[] GetTopicList()

{
try
{
// Get the user's documents folder
string userDocumentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);

// Construct the path to the TopicList.txt file

string topicListPath = Path.Combine(userDocumentsPath, "Frase.io",
"TopicList.txt");

// Check if the file exists

if (File.Exists(topicListPath))
{
// Read the topics from the file, remove empty and whitespace-
only lines
string[] topics = File.ReadAllLines(topicListPath)
.Where(line => !
string.IsNullOrWhiteSpace(line)) // Filters out empty lines
.ToArray();

return topics;
}
else
{
Console.WriteLine("TopicList.txt file does not exist.");
return Array.Empty<string>(); // Return an empty array if file
does not exist
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
return Array.Empty<string>(); // Return an empty array in case of
an error
}
}
static string GetLatestFile(string folderPath, string searchPattern)
{
try
{
var directory = new DirectoryInfo(folderPath);
var file = directory.GetFiles(searchPattern)
 .OrderByDescending(f => f.LastWriteTime)
.FirstOrDefault();
return file?.FullName;
}
catch (Exception ex)
{
Console.WriteLine($"Error getting latest file: {ex.Message}");
return null;
}
}

static void WaitForNewFiles(string folderPath, HashSet<string>

initialFiles, int timeoutInSeconds, string topicName)
{
DateTime endTime = DateTime.Now.AddSeconds(timeoutInSeconds);

while (DateTime.Now < endTime)

{
var allFiles = Directory.GetFiles(folderPath);
var newFiles = allFiles.Where(f => !
initialFiles.Contains(f)).ToList();
var downloadingFiles = newFiles.Where(f =>
f.EndsWith(".crdownload") || f.EndsWith(".part")).ToList();
var csvFiles = newFiles.Where(f => f.EndsWith(".csv")).ToList();

if (csvFiles.Any() && !downloadingFiles.Any())

{
Console.WriteLine("File downloaded successfully! Waiting for
file to be fully written...");
string downloadedFile = csvFiles.First();

// Wait until the file is no longer locked

int retries = 5;
while (retries > 0)
{
try
{
using (FileStream fs = new FileStream(downloadedFile,
FileMode.Open, FileAccess.Read, FileShare.None))
{
Console.WriteLine("File is fully available.");
return;
}
}
catch (IOException)
{
Console.WriteLine("File is still being written,
retrying...");
Thread.Sleep(2000);
retries--;
}
}
Console.WriteLine("Warning: File may still be incomplete.");
return;
}

Thread.Sleep(1000);
}
 LogError($"Files did not download for topic = '{topicName}'");
}

public static (string Country, string Language)

GetCountryAndLanguage(string filePath)
{
if (File.Exists(filePath))
{
string[] lines = File.ReadAllLines(filePath);
if (lines.Length >= 2)
return (lines[0].Trim(), lines[1].Trim());
}
return (string.Empty, string.Empty); // Return empty values if file is
missing or invalid
}

}
}