// See https://aka.ms/new-console-template for more information using Newtonsoft.Json; using System; using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Data.OleDb; using System.Data.SqlClient; using System.Diagnostics; using System.Drawing; using System.Globalization; using System.Linq; using System.Net.Http; using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Xml; using Npgsql; internal class Plat_integ { //tempo medio de execucao para processamente sincrono: 15 dias: 5,4s / 30 dias:10,0s / 60 dias:16,2s static string PG_CONN_STRING_DEV = "Server = smart-energia-dev-pgsql.cykff7tj7mik.us-east-1.rds.amazonaws.com; Port = 5432; Database = smartimptest; User Id = postgres; Password = VfHml#Z78!%kvvNM;"; static string PG_CONN_STRING_PROD = "Server = smart-energia-dev-pgsql.cykff7tj7mik.us-east-1.rds.amazonaws.com; Port = 5432; Database = smartenergiaprod; User Id = postgres; Password = VfHml#Z78!%kvvNM;"; static string ENVIRONMENT = "prod"; // "dev" | "prod" static string BUILD_FOR = "access"; // "access" | "postgres" static void Main() { MainAsync().Wait(); } private static async Task MainAsync() { string caminho_BD; int max_i; DateTime data_inicial; DateTime tempo_cod, tempo_ini; Thread.CurrentThread.CurrentCulture = CultureInfo.CreateSpecificCulture("en-GB"); //auxiliar para calculo do tempo de execucao do codigo tempo_ini = DateTime.Now; tempo_cod = DateTime.Now; //caminho do BD em que sera salvo os valores de PLD caminho_BD = @"X:\Middle\Informativo Setorial\Modelo Word\BD_SCDE.accdb"; //data inicial conforme ultimo valor salvo no BD data_inicial = data_inicio_BD(caminho_BD); DateTime data_final = DateTime.Now.AddDays(1); if (data_inicial.Year != data_final.Year) { data_final = new DateTime(data_inicial.Year, 12, 31); } //roda a requisicao de forma sincrona para obter o numero total de paginas - tambem busca os dados da primeira pagina max_i = processar_XML_sync(requisicao_http_sync(xml_requisicao(1, data_inicial, data_final)), caminho_BD); Console.Write("Tempo de execucao da etapa inicial: {0}", DateTime.Now - tempo_cod); Console.WriteLine("\nHavera {0} processos rodando em paralelo", max_i - 1); tempo_cod = DateTime.Now; if (BUILD_FOR == "postgres") { int pagesN; for (int i = 2; i <= max_i; i++) { pagesN = processar_XML_sync(requisicao_http_sync(xml_requisicao(i, data_inicial, data_final)), caminho_BD); } } else { //abre a conexao com o BD OleDbConnection access_conn = new("Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" + caminho_BD + ";Jet OLEDB:Database Password=gds21"); access_conn.Open(); await Task.WhenAll(assync_requisicao_http(data_inicial, max_i, caminho_BD, access_conn)); Thread.Sleep(2000); Console.WriteLine("Concluido. Tempo TOTAL de execucao: {0}", DateTime.Now - tempo_ini); //repete a requisicao de forma assincrona para as demais paginas access_conn.Close(); } return 0; } public static async Task assync_requisicao_http(DateTime data_inicial, int max_i, string caminho_BD, OleDbConnection access_conn) { int i; var handler = new HttpClientHandler(); Task[] arr_Tks = new Task[max_i - 1]; Task[] arr_Tks_gen = new Task[max_i - 1]; var arr_Var = new HttpResponseMessage[max_i - 1]; // Carrega o certificado do arquivo .pfx X509Certificate2 cert = new X509Certificate2("X:\\Back\\APP Smart\\Certificado\\Certificado Fernando.pfx", "appsmart"); handler.ClientCertificates.Add(cert); DateTime data_final = DateTime.Now; if (data_inicial.Year != data_final.Year) { data_final = new DateTime(data_inicial.Year, 12, 31); } DateTime aux_tempo = DateTime.Now; var client = new HttpClient(handler); client.DefaultRequestHeaders.Add("SOAPAction", "listarPLD"); var endpoint = new Uri("https://servicos.ccee.org.br:443/ws/prec/PLDBSv1"); Parallel.For(2, max_i + 1, (i) => { var payload = new StringContent(xml_requisicao(i, data_inicial, data_final), Encoding.UTF8, "application/json"); arr_Tks[i - 2] = client.PostAsync(endpoint, payload); }); Parallel.For(2, max_i + 1, (i, state) => { var retorno = arr_Tks[i - 2].Result; try { arr_Tks_gen[i - 2] = processar_XML_assync(retorno.Content.ReadAsStringAsync().Result, caminho_BD, access_conn); } catch (AggregateException ae) { Task.Delay(100); arr_Tks_gen[i - 2] = processar_XML_assync(retorno.Content.ReadAsStringAsync().Result, caminho_BD, access_conn); } }); await Task.WhenAll(arr_Tks_gen.ToArray()); return 2; } //processa as respostas de forma assincrona public static async Task processar_XML_assync(string entrada, string caminho_BD, OleDbConnection access_conn) { XmlDocument doc = new XmlDocument(); doc.LoadXml(entrada); //doc.Save(@"X:\Back\PLD Horário\xmlresposta.xml"); int hora = 0, dia_semana = 0; int j; int num_pag; double dia = 0; string submercado = "", mes = "", access_strCOM, pg_strCOM; double valor = 0; DateTime aux_data = new DateTime(2005, 01, 01); DateTime tempo_xml; XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable); nsmgr.AddNamespace("bm", "http://xmlns.energia.org.br/BM/v1"); nsmgr.AddNamespace("bo", "http://xmlns.energia.org.br/BO/v1"); nsmgr.AddNamespace("hdr", "http://xmlns.energia.org.br/MH/v1"); nsmgr.AddNamespace("soapenv", "http://schemas.xmlsoap.org/soap/envelope"); nsmgr.AddNamespace("xsi", "http://www.w3.org/2001/XMLSchema-instance"); tempo_xml = DateTime.Now; int.TryParse(doc.SelectSingleNode("//hdr:totalPaginas", nsmgr).InnerText, out num_pag); //Console.WriteLine("Total paginas " + num_pag); await Task.Run(() => { //START PGSQL NpgsqlConnection pg_conn; if (ENVIRONMENT is "dev") { pg_conn = new(PG_CONN_STRING_DEV); } else { pg_conn = new(PG_CONN_STRING_PROD); } if (BUILD_FOR != "access") { pg_conn.Open(); } foreach (XmlNode n1 in doc.SelectNodes("//bm:plds//bm:pld", nsmgr)) { XmlNode n2 = n1; aux_data = DateTime.ParseExact(n2.SelectSingleNode("bo:vigencia//bo:inicio", nsmgr).InnerText, "yyyy-MM-dd'T'HH:mm:ss'-03:00'", null); dia = (aux_data.ToOADate() - aux_data.ToOADate() % 1); hora = aux_data.Hour; mes = aux_data.ToString("yyMM"); dia_semana = (int)aux_data.DayOfWeek; if (dia_semana == 0) { dia_semana = 7; } for (j = 0; j <= 3; j++) { submercado = n2.SelectNodes("bo:valores//bo:valor//bo:submercado//bo:nome", nsmgr)[j].InnerText; Double.TryParse(n2.SelectNodes("bo:valores//bo:valor//bo:valor//bo:valor", nsmgr)[j].InnerText, out valor); if (BUILD_FOR == "access") { //START ACCESS access_strCOM = "INSERT INTO PLD_comp (Data, Hora, Submercado, Valor, Mes_ref, Dia_da_semana)"; access_strCOM += " VALUES (" + dia + "," + (hora + 1) + ",\"" + submercado + "\"," + valor + "," + mes + "," + dia_semana + ")"; OleDbCommand tbPLD = new(access_strCOM, access_conn); OleDbDataReader reader = tbPLD.ExecuteReader(); //END ACCESS } else { pg_strCOM = "INSERT INTO pld (dia_num, hora, submercado, valor, mes_ref, dia_da_semana) "; pg_strCOM += "VALUES (" + dia + ", " + (hora + 1) + ", '" + submercado + "', " + valor + ", '" + mes + "', " + dia_semana + ")"; NpgsqlCommand pg_tcSCDE = new(pg_strCOM, pg_conn); NpgsqlDataReader pg_reader = pg_tcSCDE.ExecuteReader(); pg_reader.Close(); //END PGSQL } } } pg_conn.Close(); }); foreach (XmlNode n1 in doc.GetElementsByTagName("hdr:numero")) { Console.WriteLine("Processando pagina {0} de {1} - tempo de execucao {2}", n1.InnerText, num_pag, DateTime.Now - tempo_xml); } } //prepara o xml de entrada a ser enviado à CCEE public static string xml_requisicao(int pag, DateTime dat_ini, DateTime dat_fim) { string cam_ent, tex_req, sdat_ini, sdat_fim; cam_ent = @"X:\Back\PLD Horário\listarPLD.txt"; sdat_ini = dat_ini.ToString("yyyy-MM-ddT00:00:00"); //2022-12-31T00:00:00 sdat_fim = dat_fim.ToString("yyyy-MM-ddT00:00:00"); tex_req = File.ReadAllText(cam_ent); tex_req = tex_req.Replace("DATA_INI", sdat_ini); tex_req = tex_req.Replace("DATA_FIM", sdat_fim); tex_req = tex_req.Replace("PAG_NUM", pag.ToString()); return tex_req; } //calcula a data de inicio da solicitacao do PLD, utilizando como base o ultimo dia de valor do PLD salvo no BD public static DateTime data_inicio_BD(string caminho_BD) { string access_strCOM, pg_strCOM; DateTime saida_ult_dia = DateTime.Now; if (BUILD_FOR == "access") { access_strCOM = "SELECT MAX(Data) As ultimo_dia FROM PLD_comp"; OleDbConnection access_conn = new("Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" + caminho_BD + ";Jet OLEDB:Database Password=gds21"); access_conn.Open(); OleDbCommand access_tbPLD = new(access_strCOM, access_conn); OleDbDataReader access_reader = access_tbPLD.ExecuteReader(); access_reader.Read(); DateTime.TryParse(access_reader["ultimo_dia"].ToString(), out saida_ult_dia); access_conn.Close(); saida_ult_dia = saida_ult_dia.AddDays(1); Console.WriteLine("Importando dados a partir de:" + saida_ult_dia); } else { pg_strCOM = "SELECT TO_DATE('1899-12-30', 'YYYY-MM-DD') + INTERVAL '1' DAY * MAX(dia_num) AS ultimo_dia FROM pld"; NpgsqlConnection pg_conn; if (ENVIRONMENT is "dev") { pg_conn = new(PG_CONN_STRING_DEV); } else { pg_conn = new(PG_CONN_STRING_PROD); } if (BUILD_FOR != "access") { pg_conn.Open(); } NpgsqlCommand pg_tcSCDE = new(pg_strCOM, pg_conn); NpgsqlDataReader pg_reader = pg_tcSCDE.ExecuteReader(); pg_reader.Read(); Console.WriteLine("" + pg_reader["ultimo_dia"].ToString()); if (pg_reader["ultimo_dia"].ToString().Length > 0) { DateTime.TryParse(pg_reader["ultimo_dia"].ToString(), out saida_ult_dia); pg_reader.Close(); saida_ult_dia = saida_ult_dia.AddDays(1); Console.WriteLine("Importando dados a partir de:" + saida_ult_dia); } else { saida_ult_dia = DateTime.Now.AddYears(-2); Console.WriteLine("Importando dados a partir de:" + saida_ult_dia); } } return saida_ult_dia; } //requisicao sincrona de valor do PLD public static string requisicao_http_sync(string tex_req) { var handler = new HttpClientHandler(); // Carrega o certificado do arquivo .pfx X509Certificate2 cert = new X509Certificate2("X:\\Back\\APP Smart\\Certificado\\Certificado Fernando.pfx", "appsmart"); handler.ClientCertificates.Add(cert); using (var client = new HttpClient(handler)) { client.DefaultRequestHeaders.Add("SOAPAction", "listarPLD"); var endpoint = new Uri("https://servicos.ccee.org.br:443/ws/prec/PLDBSv1"); var newPostJson = JsonConvert.ToString(tex_req); var payload = new StringContent(tex_req, Encoding.UTF8, "application/json"); //modificar a proxima linha para transformar em assincrono //var retorno = client.PostAsync(endpoint, payload).Result.Content.ReadAsStringAsync().Result; var retorno = client.PostAsync(endpoint, payload).Result.Content.ReadAsStringAsync().Result; //https://stackoverflow.com/questions/642293/how-do-i-read-and-parse-an-xml-file-in return retorno; } } //processa os dados da resposta de forma sincrona, retorna o numero total de paginas da requisicao public static int processar_XML_sync(string entrada, string caminho_BD) { XmlDocument doc = new XmlDocument(); doc.LoadXml(entrada); doc.Save(@"X:\Back\PLD Horário\xmlresposta.xml"); int hora = 0, dia_semana = 0; int num_pag; double dia = 0; string submercado = "", mes = "", access_strCOM, pg_strCOM; double valor = 0; DateTime aux_data = new DateTime(2005, 01, 01); //abre a conexao com o BD OleDbConnection access_conn = new("Provider=Microsoft.ACE.OLEDB.12.0;Data Source=" + caminho_BD + ";Jet OLEDB:Database Password=gds21"); access_conn.Open(); NpgsqlConnection pg_conn; if (ENVIRONMENT is "dev") { pg_conn = new(PG_CONN_STRING_DEV); } else { pg_conn = new(PG_CONN_STRING_PROD); } if (BUILD_FOR != "access") { pg_conn.Open(); } Thread.CurrentThread.CurrentCulture = CultureInfo.CreateSpecificCulture("en-GB"); num_pag = 1; foreach (XmlNode n1 in doc.GetElementsByTagName("hdr:totalPaginas")) { int.TryParse(n1.InnerText, out num_pag); } foreach (XmlNode n1 in doc.GetElementsByTagName("hdr:numero")) { Console.WriteLine("Processando pagina {0} de {1}", n1.InnerText, num_pag); } foreach (XmlNode n1 in doc.GetElementsByTagName("bm:pld")) { foreach (XmlNode n2 in n1.ChildNodes) { foreach (XmlNode n3 in n2.ChildNodes) { if (n3.Name == "bo:inicio") { aux_data = DateTime.ParseExact(n3.InnerText, "yyyy-MM-dd'T'HH:mm:ss'-03:00'", null); dia = (aux_data.ToOADate() - aux_data.ToOADate() % 1); hora = aux_data.Hour; mes = aux_data.ToString("yyMM"); dia_semana = (int)aux_data.DayOfWeek; if (dia_semana == 0) { dia_semana = 7; } } if (n3.Name == "bo:valor") { foreach (XmlNode n4 in n3.ChildNodes) { if (n4.Name == "bo:submercado" || n4.Name == "bo:valor") { foreach (XmlNode n5 in n4.ChildNodes) { switch (n5.Name) { case "bo:nome": submercado = n5.InnerText; break; case "bo:valor": Double.TryParse(n5.InnerText, out valor); //Console.WriteLine("Mes:" + mes + " - Dia da semana:" + dia_semana + " - Dia:" + dia + " - Hora:" + hora + "- Submercado:" + submercado + "- Valor:" + valor + "\n"); //Escreva as informacoes no BD if (BUILD_FOR == "access") { access_strCOM = "INSERT INTO PLD_comp (Data, Hora, Submercado, Valor, Mes_ref, Dia_da_semana)"; access_strCOM += " VALUES (" + dia + "," + (hora + 1) + ",\"" + submercado + "\"," + valor + "," + mes + "," + dia_semana + ")"; //Console.WriteLine(strCOM); OleDbCommand access_tbPLD = new(access_strCOM, access_conn); OleDbDataReader access_reader = access_tbPLD.ExecuteReader(); access_reader.Close(); } else { pg_strCOM = "INSERT INTO pld " + "(dia_num, hora, submercado, valor, mes_ref, dia_da_semana) "; pg_strCOM += "VALUES (" + dia + ", " + (hora + 1) + ", '" + submercado + "', " + valor + ", '" + mes + "', " + dia_semana + ")"; NpgsqlCommand pg_tcSCDE = new(pg_strCOM, pg_conn); NpgsqlDataReader pg_reader = pg_tcSCDE.ExecuteReader(); pg_reader.Close(); } break; default: break; } } } } } } } } access_conn.Close(); pg_conn.Close(); return num_pag; } }