Improve bumpUpstream: semver sort and Docker registry-based version format resolution

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dappnode/dappnodesdk",
-  "version": "0.3.45",
+  "version": "0.3.52",
   "type": "module",
   "description": "dappnodesdk is a tool to make the creation of new dappnode packages as simple as possible. It helps to initialize and publish in ethereum blockchain",
   "main": "dist/index.js",
@@ -34,7 +34,7 @@
   },
   "homepage": "https://github.com/dappnode/DAppNodeSDK#readme",
   "dependencies": {
-    "@dappnode/schemas": "^0.1.24",
+    "@dappnode/schemas": "^0.1.26",
     "@dappnode/toolkit": "^0.1.21",
     "@dappnode/types": "^0.1.41",
     "@octokit/rest": "^20.1.1",

src/commands/githubActions/bumpUpstream/github/fetchGithubUpstreamVersion.ts

@@ -1,3 +1,4 @@
+import semver from "semver";
 import { Github } from "../../../../providers/github/Github.js";
 import { isValidRelease } from "./isValidRelease.js";
 
@@ -6,9 +7,9 @@ export async function fetchGithubUpstreamVersion(
 ): Promise<string | null> {
   try {
     const newVersion = await fetchGithubLatestTag(repo);
-    if (!isValidRelease(newVersion)) {
+    if (!newVersion) {
       console.log(
-        `This is not a valid release (probably a release candidate) - ${repo}: ${newVersion}`
+        `No valid release found (probably all are release candidates) - ${repo}`
       );
       return null;
     }
@@ -21,13 +22,42 @@ export async function fetchGithubUpstreamVersion(
   }
 }
 
-async function fetchGithubLatestTag(repo: string): Promise<string> {
+async function fetchGithubLatestTag(repo: string): Promise<string | null> {
   const [owner, repoName] = repo.split("/");
   const githubRepo = new Github({ owner, repo: repoName });
 
   const releases = await githubRepo.listReleases();
-  const latestRelease = releases[0];
-  if (!latestRelease) throw Error(`No release found for ${repo}`);
 
-  return latestRelease.tag_name;
+  if (!releases?.length) {
+    throw Error(`No releases found for ${repo}`);
+  }
+
+  // Filter valid releases: not draft, not prerelease, passes semver validation
+  const validReleases = releases.filter(
+    release =>
+      !release.draft && !release.prerelease && isValidRelease(release.tag_name)
+  );
+
+  if (validReleases.length === 0) {
+    return null;
+  }
+
+  // Sort by semver descending to get the highest version
+  validReleases.sort((a, b) => {
+    const versionA = stripTagPrefix(a.tag_name);
+    const versionB = stripTagPrefix(b.tag_name);
+    if (!versionA || !versionB) return 0;
+    return semver.rcompare(versionA, versionB);
+  });
+
+  return validReleases[0].tag_name;
+}
+
+/**
+ * Strips any prefix from a tag name to extract a clean semver version.
+ * e.g. "v1.2.3" -> "1.2.3", "n8n@2.10.3" -> "2.10.3"
+ */
+function stripTagPrefix(tag: string): string | null {
+  const match = tag.match(/(\d+\.\d+\.\d+.*)$/);
+  return match ? match[1] : null;
 }

src/commands/githubActions/bumpUpstream/github/isValidRelease.ts

@@ -4,8 +4,11 @@ export function isValidRelease(version: string): boolean {
   // Nightly builds are not considered valid releases (not taken into account by semver)
   if (version.includes("nightly")) return false;
 
-  if (semver.valid(version)) {
-    const preReleases = semver.prerelease(version);
+  // Strip any prefix (e.g. "v1.2.3" -> "1.2.3", "n8n@2.10.3" -> "2.10.3")
+  const cleaned = stripTagPrefix(version) || version;
+
+  if (semver.valid(cleaned)) {
+    const preReleases = semver.prerelease(cleaned);
 
     // A version is considered a valid release if it has no pre-release components.
     return preReleases === null || preReleases.length === 0;
@@ -17,3 +20,8 @@ export function isValidRelease(version: string): boolean {
 
   return true;
 }
+
+function stripTagPrefix(tag: string): string | null {
+  const match = tag.match(/(\d+\.\d+\.\d+.*)$/);
+  return match ? match[1] : null;
+}

src/commands/githubActions/bumpUpstream/github/resolveVersionFormat.ts

@@ -0,0 +1,137 @@
+import fs from "fs";
+import path from "path";
+import { Compose } from "@dappnode/types";
+
+/**
+ * Given a GitHub release tag (e.g. "v1.17.0", "n8n@2.10.3"), resolves the
+ * correct version format by checking the upstream Docker image registry.
+ *
+ * 1. Finds which compose service uses the given build arg
+ * 2. Parses the Dockerfile to extract the Docker image that uses that arg
+ * 3. Checks the Docker registry for tag existence (with/without prefix)
+ * 4. Returns the version in the format that matches the Docker registry
+ */
+export async function resolveVersionFormat({
+  tag,
+  arg,
+  compose,
+  dir
+}: {
+  tag: string;
+  arg: string;
+  compose: Compose;
+  dir: string;
+}): Promise<string> {
+  const stripped = stripTagPrefix(tag);
+  if (!stripped || stripped === tag) return tag; // No prefix to strip
+
+  try {
+    const dockerImage = getDockerImageForArg(compose, arg, dir);
+    if (!dockerImage) return tag;
+
+    const tagExists = await checkDockerTagExists(dockerImage, stripped);
+    if (tagExists) return stripped;
+
+    return tag;
+  } catch (e) {
+    console.warn(`Could not resolve version format for ${tag}, using as-is:`, e.message);
+    return tag;
+  }
+}
+
+/**
+ * Finds the Docker image that uses a given build arg by parsing the Dockerfile.
+ */
+function getDockerImageForArg(
+  compose: Compose,
+  arg: string,
+  dir: string
+): string | null {
+  for (const [, service] of Object.entries(compose.services)) {
+    if (
+      typeof service.build !== "string" &&
+      service.build?.args &&
+      arg in service.build.args
+    ) {
+      const buildContext = service.build.context || ".";
+      const dockerfileName = service.build.dockerfile || "Dockerfile";
+      const dockerfilePath = path.resolve(dir, buildContext, dockerfileName);
+
+      if (!fs.existsSync(dockerfilePath)) continue;
+
+      const content = fs.readFileSync(dockerfilePath, "utf-8");
+      return extractImageForArg(content, arg);
+    }
+  }
+  return null;
+}
+
+/**
+ * Parses a Dockerfile to find the FROM line that references the given ARG,
+ * and extracts the Docker image name (without the tag).
+ *
+ * Handles patterns like:
+ *   FROM ethereum/client-go:${UPSTREAM_VERSION}
+ *   FROM ethereum/client-go:v${UPSTREAM_VERSION}
+ *   FROM ollama/ollama:${OLLAMA_VERSION#v}
+ *   FROM statusim/nimbus-eth2:multiarch-${UPSTREAM_VERSION}
+ */
+function extractImageForArg(
+  dockerfileContent: string,
+  arg: string
+): string | null {
+  const lines = dockerfileContent.split("\n");
+
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith("FROM") || !trimmed.includes(arg)) continue;
+
+    // Match: FROM image:tag_pattern (with optional "AS stage")
+    const match = trimmed.match(/^FROM\s+([^:\s]+)/i);
+    if (match) return match[1];
+  }
+
+  return null;
+}
+
+/**
+ * Checks if a tag exists on a Docker registry using the Docker Hub v2 API.
+ * Supports Docker Hub, ghcr.io, and gcr.io.
+ */
+async function checkDockerTagExists(
+  image: string,
+  tag: string
+): Promise<boolean> {
+  const url = getRegistryTagUrl(image, tag);
+  if (!url) return false;
+
+  try {
+    const response = await fetch(url);
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+function getRegistryTagUrl(image: string, tag: string): string | null {
+  // ghcr.io/org/image -> GitHub Container Registry
+  if (image.startsWith("ghcr.io/")) {
+    const imagePath = image.replace("ghcr.io/", "");
+    return `https://ghcr.io/v2/${imagePath}/manifests/${tag}`;
+  }
+
+  // gcr.io/project/image -> Google Container Registry
+  if (image.startsWith("gcr.io/")) {
+    const imagePath = image.replace("gcr.io/", "");
+    return `https://gcr.io/v2/${imagePath}/manifests/${tag}`;
+  }
+
+  // Docker Hub: library/image or org/image
+  const dockerImage = image.includes("/") ? image : `library/${image}`;
+  return `https://registry.hub.docker.com/v2/repositories/${dockerImage}/tags/${tag}`;
+}
+
+function stripTagPrefix(tag: string): string | null {
+  const match = tag.match(/(\d+\.\d+\.\d+.*)$/);
+  return match ? match[1] : null;
+}

src/commands/githubActions/bumpUpstream/settings/getInitialSettings.ts

@@ -1,9 +1,10 @@
-import { Manifest, UpstreamItem } from "@dappnode/types";
+import { Manifest, UpstreamItem, Compose } from "@dappnode/types";
 import { readManifest, readCompose } from "../../../../files/index.js";
 import { arrIsUnique } from "../../../../utils/array.js";
 import { getFirstAvailableEthProvider } from "../../../../utils/tryEthProviders.js";
 import { InitialSetupData, GitSettings, UpstreamSettings } from "../types.js";
 import { fetchGithubUpstreamVersion } from "../github/fetchGithubUpstreamVersion.js";
+import { resolveVersionFormat } from "../github/resolveVersionFormat.js";
 
 export async function getInitialSettings({
   dir,
@@ -17,7 +18,7 @@ export async function getInitialSettings({
   const { manifest, format } = readManifest([{ dir }]);
   const compose = readCompose([{ dir }]);
 
-  const upstreamSettings = await parseUpstreamSettings(manifest);
+  const upstreamSettings = await parseUpstreamSettings(manifest, compose, dir);
 
   const gitSettings = getGitSettings();
 
@@ -44,11 +45,13 @@ export async function getInitialSettings({
  * field (array of objects with 'repo', 'arg' and 'version' fields)
  */
 async function parseUpstreamSettings(
-  manifest: Manifest
+  manifest: Manifest,
+  compose: Compose,
+  dir: string
 ): Promise<UpstreamSettings[] | null> {
   const upstreamSettings = manifest.upstream
-    ? await parseUpstreamSettingsNewFormat(manifest.upstream)
-    : await parseUpstreamSettingsLegacyFormat(manifest);
+    ? await parseUpstreamSettingsNewFormat(manifest.upstream, compose, dir)
+    : await parseUpstreamSettingsLegacyFormat(manifest, compose, dir);
 
   if (!upstreamSettings || upstreamSettings.length < 1) return null;
 
@@ -58,13 +61,22 @@ async function parseUpstreamSettings(
 }
 
 async function parseUpstreamSettingsNewFormat(
-  upstream: UpstreamItem[]
+  upstream: UpstreamItem[],
+  compose: Compose,
+  dir: string
 ): Promise<UpstreamSettings[]> {
   const upstreamPromises = upstream.map(async ({ repo, arg, version }) => {
     const githubVersion = await fetchGithubUpstreamVersion(repo);
 
-    if (githubVersion)
-      return { repo, arg, manifestVersion: version, githubVersion };
+    if (githubVersion) {
+      const resolvedVersion = await resolveVersionFormat({
+        tag: githubVersion,
+        arg,
+        compose,
+        dir
+      });
+      return { repo, arg, manifestVersion: version, githubVersion: resolvedVersion };
+    }
   });
 
   const upstreamResults = await Promise.all(upstreamPromises);
@@ -78,7 +90,9 @@ async function parseUpstreamSettingsNewFormat(
  * Currently, 'upstream' field is used instead, which is an array of objects with 'repo', 'arg' and 'version' fields
  */
 async function parseUpstreamSettingsLegacyFormat(
-  manifest: Manifest
+  manifest: Manifest,
+  compose: Compose,
+  dir: string
 ): Promise<UpstreamSettings[] | null> {
   // 'upstreamRepo' and 'upstreamArg' being defined as arrays has been deprecated
 
@@ -89,12 +103,20 @@ async function parseUpstreamSettingsLegacyFormat(
 
   if (!githubVersion) return null;
 
+  const arg = manifest.upstreamArg || "UPSTREAM_VERSION";
+  const resolvedVersion = await resolveVersionFormat({
+    tag: githubVersion,
+    arg,
+    compose,
+    dir
+  });
+
   return [
     {
       repo: manifest.upstreamRepo,
       manifestVersion: manifest.upstreamVersion || "UPSTREAM_VERSION",
-      arg: manifest.upstreamArg || "UPSTREAM_VERSION",
-      githubVersion
+      arg,
+      githubVersion: resolvedVersion
     }
   ];
 }

src/providers/github/Github.ts

@@ -137,6 +137,7 @@ export class Github {
       .listReleases({
         owner: this.owner,
         repo: this.repo,
+        per_page: 100,
         headers: {
           "X-GitHub-Api-Version": "2022-11-28"
         }
@@ -231,7 +232,7 @@ export class Github {
     assetsDir: string;
     matchPattern?: RegExp;
     fileNamePrefix?: string;
-  }) {
+  }): Promise<void> {
     for (const file of fs.readdirSync(assetsDir)) {
       // Used to ignore duplicated legacy .tar.xz image
       if (matchPattern && !matchPattern.test(file)) continue;
@@ -246,7 +247,7 @@ export class Github {
               owner: this.owner,
               repo: this.repo,
               release_id: releaseId,
-              data: fs.createReadStream(filepath) as any,
+              data: fs.createReadStream(filepath) as unknown as string,
               headers: {
                 "content-type": contentType,
                 "content-length": fs.statSync(filepath).size