Merge pull request #185 from Lemoncode/feature/mino-scraping

Feature/mino scraping

Author: antonio06

front/next-env.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
-import "./.next/dev/types/routes.d.ts";
+import "./.next/types/routes.d.ts";
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

functions/src/functions/scraping-functions.ts

@@ -36,6 +36,8 @@ export async function scrapingsFunction(
 
     const responseCuencaSegura = await embalsesRepository.actualizarCuencaSegura();
 
+    const responseCuencaMinoSil = await embalsesRepository.actualizarCuencaMinoSil();
+
     if (responseCuencaMediterranea) {
       context.log(
         "scrapings-function: Se han actualizado los embalses de la cuenca Mediterránea",
@@ -92,6 +94,14 @@ export async function scrapingsFunction(
       );
     }
 
+    if (responseCuencaMinoSil) {
+      context.log(`Se han actualizado los embalses de la cuenca Mino Sil`);
+    } else {
+      context.log(
+        "No se han podido actualizar los embalses de la cuenca Mino Sil"
+      );
+    }
+
   } catch (error) {
     context.error("scrapings-function: ERROR", error);
     throw error;

integrations/scraping-cuenca-mino-sil/package.json

@@ -4,12 +4,20 @@
   "private": true,
   "type": "module",
   "exports": {
-    ".": "./src/index.ts"
+    ".": "./dist/index.js"
   },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "scripts": {
-    "start": "tsx --watch src/console-runner.ts"
+    "start": "tsx --watch src/console-runner.ts",
+    "build": "run-p clean type-check build:scraping-cuenca-mino-sil",
+    "build:scraping-cuenca-mino-sil": "tsc",
+    "clean": "rimraf dist",
+    "type-check": "tsc --noEmit --preserveWatchOutput"
   },
   "dependencies": {
+    "axios": "^1.7.0",
+    "cheerio": "^1.1.2",
     "db-model": "^1.0.0"
   }
 }

integrations/scraping-cuenca-mino-sil/src/api/cuenca.api.ts

@@ -0,0 +1,47 @@
+import axios from "axios";
+import https from "https";
+
+const httpsAgent = new https.Agent({
+  rejectUnauthorized: false,
+});
+
+const browserHeaders = {
+  "User-Agent":
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+  Accept:
+    "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+  "Accept-Language": "es-ES,es;q=0.9,en;q=0.8",
+};
+
+const BASE_URL =
+  "https://saih.chminosil.es/index.php?url=/datos/mapas/mapa:H1/area:HID/acc:1";
+const TARGET_URL =
+  "https://saih.chminosil.es/index.php?url=/datos/situacionEmbalses";
+
+export async function getCuencaPageHTMLContent(): Promise<string> {
+  const sessionResponse = await axios.get(BASE_URL, {
+    httpsAgent,
+    maxRedirects: 0,
+    headers: browserHeaders,
+    validateStatus: (status) => status >= 200 && status < 400,
+  });
+
+  const setCookieHeaders = sessionResponse.headers["set-cookie"];
+  const cookieString = setCookieHeaders
+    ? setCookieHeaders
+      .map((cookie: string) => cookie.split(";")[0])
+      .join("; ")
+    : "";
+
+  const { data: html } = await axios.get(TARGET_URL, {
+    httpsAgent,
+    maxRedirects: 10,
+    headers: {
+      ...browserHeaders,
+      Cookie: cookieString,
+      Referer: BASE_URL,
+    },
+  });
+
+  return html;
+}

integrations/scraping-cuenca-mino-sil/src/api/cuenca.model.ts

@@ -0,0 +1,7 @@
+export interface EmbalsesMinoSil {
+  id: number;
+  embalse: string;
+  capacidadTotalHm3: number;
+  volumenActualHm3: number;
+  fecha: string;
+}

integrations/scraping-cuenca-mino-sil/src/api/index.ts

@@ -0,0 +1,2 @@
+export * from "./cuenca.api.js";
+export * from "./cuenca.model.js";

integrations/scraping-cuenca-mino-sil/src/console-runner.ts

@@ -1,5 +1,5 @@
-import { getEstadoCuencaMinoSil } from "./integration";
+import { scrapeCuencaMinioSil } from "./integration.js";
 
 console.log("Estado de la Cuenca Miño Sil:");
-const result = await getEstadoCuencaMinoSil();
-console.log(JSON.stringify(result, null, 2));
+const result = await scrapeCuencaMinioSil();
+console.log(result);

integrations/scraping-cuenca-mino-sil/src/index.ts

@@ -1,2 +1,2 @@
-export * from "./integration";
+export * from "./integration.js";
 

integrations/scraping-cuenca-mino-sil/src/integration.ts

@@ -1,24 +1,13 @@
-import type { Embalse } from "db-model";
+import * as cheerio from "cheerio";
+import { EmbalseUpdateSAIHEntity } from "db-model";
+import { getCuencaPageHTMLContent } from "./api/index.js";
+import { extractProvinceTables } from "./scraper/index.js";
+import { mapToEmbalseUpdateSAIH } from './scraper/index.js';
 
-export const getEstadoCuencaMinoSil = async (): Promise<Embalse[]> => {
-  return [
-    {
-      id: "1",
-      nombre: "Embalse de Belesar",
-      provincia: "Lugo",
-      capacidad: 3000000000,
-      nivelActual: 2500000000,
-      fechaUltimoNivel: new Date("2023-10-01"),
-      porcentajeLlenado: 83.3,
-    },
-    {
-      id: "2",
-      nombre: "Embalse de Velle",
-      provincia: "Ourense",
-      capacidad: 500000000,
-      nivelActual: 400000000,
-      fechaUltimoNivel: new Date("2023-10-01"),
-      porcentajeLlenado: 80.0,
-    },
-  ];
-};
+export async function scrapeCuencaMinioSil(): Promise<EmbalseUpdateSAIHEntity[]> {
+  const html = await getCuencaPageHTMLContent();
+  const $: cheerio.CheerioAPI = cheerio.load(html);
+  const rawEmbalses = extractProvinceTables($);
+
+  return mapToEmbalseUpdateSAIH(rawEmbalses);
+}

integrations/scraping-cuenca-mino-sil/src/scraper/business.ts

@@ -0,0 +1,44 @@
+import * as cheerio from "cheerio";
+import { EmbalsesMinoSil } from '../api/index.js';
+import { mapStringToApiDate } from './helpers.js';
+
+export function parseEuropeanNumber(value: string): number {
+  if (!value || value.trim() === "" || value === "*" || value === "n/d") {
+    return NaN;
+  }
+
+  // Replace comma with dot for decimal separator
+  const normalizedValue = value.replace(",", ".");
+  return parseFloat(normalizedValue);
+}
+
+export function extractProvinceTables(
+  $: cheerio.CheerioAPI
+): EmbalsesMinoSil[] {
+
+  let embalses: EmbalsesMinoSil[] = [];
+
+  $("table.tabla tr").each((_index, row) => {
+    const cells = $(row).find("td");
+    if (cells.length === 0) return; // Skip header rows with <th>
+
+    const nombre = $(cells[1]).text().trim();
+    if (!nombre || !nombre.includes(" - ")) return; // Skip totals row
+    const id = nombre.split(" - ")[0].trim().match(/^([A-Z])(\d+)$/)[2]; // Extract ID from name
+    const capacidadTotal = $(cells[4]).text().trim();
+    const fecha = $(cells[8]).text().trim();
+    const volumenActual = $(cells[6]).text().trim();
+
+    const embalse: EmbalsesMinoSil = {
+      id: Number(id),
+      embalse: nombre.split(" - ")[1].trim(),
+      capacidadTotalHm3: parseEuropeanNumber(capacidadTotal),
+      volumenActualHm3: parseEuropeanNumber(volumenActual),
+      fecha: mapStringToApiDate(fecha)
+    };
+
+    embalses = [...embalses, embalse];
+  });
+
+  return embalses;
+}