InfoSetorial/Program.cs

using System.Data.OleDb;
using System.Diagnostics;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading.Tasks.Dataflow;
using System.Timers;
using Microsoft.Extensions.Configuration;

namespace InfoSetorial
{
    public class Program
    {
        static DateTime endTime;
        static string? CCEE_URL;
        static string? AccessPath;
        static string? ResourcesTable;
        static string? str_connection;
        static string? CloseTime;

        static void SetEndTime(DateTime date) => endTime = date;
        static void SetTimer()
        {
            var timer = new System.Timers.Timer(100);
            double remainingTime;
            int interval = Convert.ToInt32(CloseTime!);
            SetEndTime(DateTime.Now.AddSeconds(interval));
            timer.Elapsed += EnquantoTimerRodar!;
            timer.AutoReset = true;
            timer.Enabled = true;

            Console.WriteLine("");
            Console.WriteLine("Pressione qualquer tecla para encerrar o programa...");
            Console.WriteLine("");

            Task.Factory.StartNew(
                () =>
                {
                    Console.ReadKey();
                    remainingTime = DateTime.Now.Subtract(endTime).TotalSeconds;
                }
            ).Wait(
                TimeSpan.FromSeconds(interval)
            );
        }
        static void SetConfig()
        {
            IConfigurationRoot config;
            config = new ConfigurationBuilder()
                .AddUserSecrets<Program>()
                .Build();
            
            CCEE_URL = config["CCEE_URL"]!;
            AccessPath = config["AccessPath"]!;
            ResourcesTable = config["ResourcesTable"]!;
            CloseTime = config["CloseTime"];
            str_connection = $"Provider=Microsoft.ACE.OLEDB.12.0;Data Source={AccessPath};Persist Security Info=False;";
            
            if (config.AsEnumerable().Where(c => c.Value == "").ToList().Any()) { throw new Exception("Falha ao acessar o arquivo de configuração"); }
        }
        private static async Task Main()
        {
            try
            {
                SetConfig();
                int offset = 0;
                int count = 0;
                using OleDbConnection connection = new(str_connection);
                connection.Open();

                if (!TabelaExiste(connection, ResourcesTable!)) { throw new Exception("Falha ao acessar [tblResourceInfoSetorial]"); }

                using OleDbCommand command = new($"SELECT {ResourcesTable}.resource_id, *\r\nFROM {ResourcesTable}\r\nWHERE ((Not ({ResourcesTable}.resource_id)=\"\"));\r\n", connection);
                using OleDbDataReader reader = command.ExecuteReader() ??  throw new Exception();

                while (reader.Read())
                {
                    offset = 0;
                    count = 0;
                    string TableName = $"tbl_{reader["dataset"]}";

                    if (TabelaExiste(connection, TableName))
                    {
                        using OleDbCommand cmd = new($"SELECT Count([_id]) FROM {TableName} HAVING (((Left([MES_REFERENCIA],4))=\"{reader["Ano"].ToString()}\"))", connection);
                        if (cmd.ExecuteScalar()!.ToString() != "")
                        {
                            count = (int)cmd.ExecuteScalar()!;
                        }
                    }

                    string webString = $@"{CCEE_URL}?";

                    for (int iCol = 0; iCol < reader.FieldCount; iCol++)
                    {
                        if (reader.GetName(iCol) != "dataset" && reader.GetName(iCol) != "Ano")
                        {
                            if (reader[iCol].ToString() != "")
                            {
                                webString = $@"{webString}{reader.GetName(iCol)}={reader[iCol]}&";
                            }
                        }
                    }

                    bool hasNextPage = true;
                    JsonNode? tempRecords = null;

                    while (hasNextPage)
                    {
                        JsonNode? rootObject = await PullDataAsync($@"{webString}offset={offset}");
                        tempRecords = rootObject["result"]!["records"]!;

                        int iTotal = (int)rootObject!["result"]!["total"]!;
                        int iFirst = tempRecords.AsArray().Count;

                        if (iFirst > (iTotal - count)) { iFirst = (iTotal - count); }

                        if (tempRecords != null && iFirst != 0 && count < iTotal)
                        {
                            var test = JsonSerializer.Serialize(tempRecords.AsArray().Take(iFirst)).ToString();
                            AdicionarDadosAoBanco(connection, (JsonNode)JsonDocument.Parse(test).Deserialize(typeof(JsonNode))!, TableName);

                            offset = iFirst + offset;
                            count = iFirst + count;
                        }
                        else
                        {
                            Console.WriteLine("Nada mais a adicionar.");
                            hasNextPage = false;
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
            finally
            {
                //SetTimer();
                Process.GetCurrentProcess().Kill();
            }

        }
        private static void EnquantoTimerRodar(object sender, ElapsedEventArgs e)
        {
            Console.Write($"\rEncerrando em {endTime.Subtract(e.SignalTime).Seconds}");
        }
        private static async Task<JsonNode> PullDataAsync(string sURL)
        {
            var client = new HttpClient
            {
                Timeout = TimeSpan.FromSeconds(1000)
            };
            var request = new HttpRequestMessage(HttpMethod.Get, sURL);
            var response = await client.SendAsync(request);
            response.EnsureSuccessStatusCode();

            JsonNode? rootObject = JsonSerializer.Deserialize<JsonNode?>(await response.Content.ReadAsStringAsync())!;

            return rootObject;
        }
        private static void AdicionarDadosAoBanco(OleDbConnection connection, JsonNode records, string TableName)
        {
            try
            {
                string stringColumns = String.Join(", \n", records[0]!.AsObject().AsEnumerable().Select(d => "@" + d.Key).ToList());

                if (!TabelaExiste(connection, TableName))
                {
                    string createColumns = String.Join(", \n", records[0]!.AsObject().AsEnumerable().Select(d => d.Key + " TEXT").ToList());
                    CriarTabela(connection, TableName, createColumns);
                }

                using OleDbTransaction transaction = connection.BeginTransaction();
                try
                {
                    int iRows = 0;
                    iRows = records!.AsArray().Count;
                    
                    for (int iRow = 0; iRow < iRows; iRow++)
                    {
                        var record = records[iRow];

                        using OleDbCommand command = new(@$"INSERT INTO {TableName} VALUES ({stringColumns})", connection, transaction);

                        foreach (var child in record!.AsObject())
                        {
                            command.Parameters.AddWithValue(child.Key, (child.Value ?? "")!.ToString());
                        }

                        command.ExecuteNonQuery();
                    }

                    transaction.Commit();
                    Console.WriteLine($"{records!.AsArray().Count} dados adicionados com sucesso à [{TableName}]");
                }
                catch (Exception ex)
                {
                    transaction.Rollback();
                    Console.WriteLine($"Erro ao adicionar dados: {ex.Message}");
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Ocorreu um erro: {ex.Message}");
            }
        }

        private static bool TabelaExiste(OleDbConnection connection, string TableName)
        {
            try
            {
                using OleDbCommand command = new(@$"SELECT COUNT(*) FROM {TableName}", connection);
                command.ExecuteScalar();
                return true;
            }
            catch (OleDbException)
            {
                return false;
            }
        }

        private static void CriarTabela(OleDbConnection connection, string TableName, string columnNames)
        {
            try
            {
                using OleDbCommand command = new(@$"CREATE TABLE {TableName} ({columnNames})", connection);
                command.ExecuteNonQuery();
            }
            catch (Exception ex)
            {
                Console.WriteLine($"Ocorreu um erro: {ex.Message}");
            }
        }
    }
}