feat: Add first structure of replicator service

.env

@@ -0,0 +1,7 @@
+POSTGRES_HOST=localhost
+POSTGRES_PORT=5433
+POSTGRES_USER=user
+POSTGRES_PASSWORD=password
+POSTGRES_DB=postgres
+
+DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable

.env.example

@@ -0,0 +1,8 @@
+POSTGRES_HOST=localhost // hostname of the PostgreSQL server
+POSTGRES_PORT=5433 // port number of the PostgreSQL server
+POSTGRES_USER=user // username for the PostgreSQL database
+POSTGRES_PASSWORD=password // password for the PostgreSQL database
+POSTGRES_DB=postgres // name of the PostgreSQL database
+
+# Database configuration (2/2) - Remember to set sslmode for production
+DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable

.gitignore

@@ -1,8 +1,5 @@
-
-.PHONY: help
-
-
-
-help:
-    @echo "Available commands:"
-    @echo "  make help        - Show this help message"
+.env
+bin
+bin/*
+tmp
+tmp/*

Makefile

@@ -16,11 +16,36 @@ dropBin:
 replicator-service:
 	air -c configs/air/replicator-service.toml
 
+.PHONY: migrate-up
+migrate-up:
+	migrate -path migrations -database $(DB_URL) up
+
+.PHONY: migrate-down
+migrate-down:
+	migrate -path migrations -database $(DB_URL) down
+
+.PHONY: migrate-force
+migrate-force:
+	migrate -path migrations -database $(DB_URL) force $(version)
+
+.PHONY: complete-testing-npm-db
+complete-testing-npm-db:
+	bash scripts/completeNpmDb.sh
+
+.PHONY: create-testing-npm-db
+create-testing-npm-db:
+	bash scripts/createTestingNpmDb.sh
+
 .PHONY: help
 help:
 	@echo "Available commands:"
-	@echo "make help               - Show this help message"
-	@echo "make init               - Initialize pre-push hook"
-	@echo "make build              - Build the project"
-	@echo "make dropBin     	   - Drop the binary files"
-	@echo "make replicator-service - Run the replicator service with air in development mode" 
+	@echo "make help                   	   - Show this help message"
+	@echo "make init                   	   - Initialize pre-push hook"
+	@echo "make build                  	   - Build the project"
+	@echo "make dropBin                    - Drop the binary files"
+	@echo "make replicator-service         - Run the replicator service with air in development mode" 
+	@echo "make migrate-up                 - Run database migrations up"
+	@echo "make migrate-down               - Run database migrations down"
+	@echo "make migrate-force              - Force database migration to a specific version (usage: make migrate-force version=<version_number>)"
+	@echo "make complete-testing-npm-db    - Complete testing npm database"
+	@echo "make create-testing-npm-db      - Create testing npm database"

README.md

@@ -8,8 +8,8 @@ API for tracking vulnerability occurrence frequencies.
 
 ```bash
 make init
-2. Install Go
 ```
+2. Install Go
 
 This step depends on your operating system.
 
@@ -51,3 +51,17 @@ go install github.com/air-verse/air@latest
 ```bash
 make replicator-service
 ```
+
+4. pick up Postgres db
+
+- podman
+
+```bash
+podman compose up -d
+```
+
+ - or docker
+
+```bash
+docker compose up -d
+```

cmd/replicator-service/main.go

@@ -1,5 +1,12 @@
 package main
 
+import (
+	"github.com/trustpkg/trustpkg-api/db"
+	"github.com/trustpkg/trustpkg-api/internal/npm"
+)
+
 func main() {
-	println("Hello, World!")
-}
+	db.ConnectDb()
+
+	npm.Pipeline()
+}

db/postgres.go

@@ -0,0 +1,65 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	goDotEnv "github.com/joho/godotenv"
+)
+
+type postgresConfig struct {
+	host     string
+	port     string
+	user     string
+	password string
+	dbName   string
+}
+
+var Pool *pgxpool.Pool
+
+func ConnectDb() {
+	if err := goDotEnv.Load(".env"); err != nil {
+		log.Println(".env file not found, using system environment variables")
+	}
+
+	dsn := strings.TrimSpace(os.Getenv("DB_URL"))
+	if dsn == "" {
+		var dbConfig = postgresConfig{
+			host:     os.Getenv("POSTGRES_HOST"),
+			port:     os.Getenv("POSTGRES_PORT"),
+			user:     os.Getenv("POSTGRES_USER"),
+			password: os.Getenv("POSTGRES_PASSWORD"),
+			dbName:   os.Getenv("POSTGRES_DB"),
+		}
+
+		if dbConfig.host == "" || dbConfig.port == "" || dbConfig.user == "" || dbConfig.dbName == "" {
+			fmt.Fprintln(os.Stderr, "Missing DB config. Set DB_URL or POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB (copy .env.example to .env).")
+			os.Exit(1)
+		}
+
+		dsn = fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", dbConfig.user, dbConfig.password, dbConfig.host, dbConfig.port, dbConfig.dbName)
+	}
+
+	pool, err := pgxpool.New(context.Background(), dsn)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err)
+		os.Exit(1)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err = pool.Ping(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Unable to ping database: %v\n", err)
+		os.Exit(1)
+	}
+
+	Pool = pool
+
+	log.Println("Connected to postgres database successfully")
+}

docker-compose.couchDb.yml

@@ -0,0 +1,11 @@
+services:
+  couchdb:
+    image: docker.io/library/couchdb:latest
+    restart: always
+    ports:
+      - 3200:5984
+    environment:
+      COUCHDB_USER: admin
+      COUCHDB_PASSWORD: password
+      TZ: UTC
+

docker-compose.yml

@@ -21,7 +21,7 @@ services:
       - 8081:8080
     depends_on: 
       - db
-    network:
+    networks:
       - trustpkg-net
 
 volumes:

go.mod

@@ -1,3 +1,16 @@
-module github.com/TymekGluch/trustpkg-api
+module github.com/trustpkg/trustpkg-api
 
 go 1.26.5
+
+require (
+	github.com/jackc/pgx/v5 v5.10.0
+	github.com/joho/godotenv v1.5.1
+)
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/text v0.29.0 // indirect
+)

go.sum

@@ -0,0 +1,28 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
+github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/npm/constants.go

@@ -0,0 +1,8 @@
+package npm
+
+const (
+	npmUrl            = "https://replicate.npmjs.com"
+	limit             = "1000"
+	shouldIncludeDocs = "true"
+	rest              = "/_changes?since=123456789&limit=1000&include_docs=true"
+)

internal/npm/http.go

@@ -0,0 +1,40 @@
+package npm
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+)
+
+func fetchNpmPackagesByCurrentSequence(ctx context.Context) error {
+	client := &http.Client{}
+
+	sequence, err := selectLastSequence(ctx)
+	if err != nil {
+		return err
+	}
+
+	url := npmUrl + "_changes?since=" + strconv.Itoa(sequence) + "&limit=" + limit + "&include_docs=" + shouldIncludeDocs
+
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+
+	request.Header.Set("Accept-Encoding", "gzip")
+
+	response, err := client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	fmt.Println("response: ", response)
+
+	if response.StatusCode == 200 {
+		insertLastSequence(ctx, sequence)
+	}
+
+	return nil
+}

internal/npm/models.go

@@ -0,0 +1 @@
+package npm

internal/npm/replicator.go

@@ -0,0 +1,2 @@
+package npm
+

internal/npm/repository.go

@@ -0,0 +1,31 @@
+package npm
+
+import (
+	"context"
+
+	"github.com/trustpkg/trustpkg-api/db"
+)
+
+var pool = db.Pool
+
+const (
+	queryInsertLastSequence = `INSERT INTO last_sequence (sequence) VALUES ($1)`
+	querySelectLastSequence = `SELECT sequence FROM last_sequence ORDER BY sequence DESC LIMIT 1`
+)
+
+func insertLastSequence(ctx context.Context, sequence int) error {
+	_, err := pool.Exec(ctx, queryInsertLastSequence, sequence)
+
+	return err
+}
+
+func selectLastSequence(ctx context.Context) (int, error) {
+	var sequence int
+
+	err := pool.QueryRow(ctx, querySelectLastSequence).Scan(&sequence)
+	if err != nil {
+		return 0, err
+	}
+
+	return sequence, nil
+}

internal/npm/service.go

@@ -0,0 +1,9 @@
+package npm
+
+import "context"
+
+func Pipeline() {
+	ctx := context.Background()
+
+	fetchNpmPackagesByCurrentSequence(ctx)
+}

migrations/0001_create_packages_table.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS packages;
+
+DROP INDEX IF EXISTS idx_packages_name;

migrations/0001_create_packages_table.up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS packages (
+    id BIGSERIAL PRIMARY KEY UNIQUE,
+    name TEXT NOT NULL UNIQUE,
+    version TEXT NOT NULL,
+    description TEXT,
+    license TEXT,
+    author TEXT,
+    repository TEXT,
+    created_at TIMESTAMP DEFAULT NOW(),
+    updated_at TIMESTAMP DEFAULT NULL,
+    maintainers TEXT[],
+    keywords TEXT[],
+    deprecated BOOLEAN DEFAULT FALSE
+);
+
+CREATE INDEX IF NOT EXISTS idx_packages_name ON packages(LOWER(name));

migrations/0002_create_versions_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS versions;

migrations/0002_create_versions_table.up.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS versions (
+    id BIGSERIAL PRIMARY KEY,
+    package_id BIGINT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
+    version TEXT NOT NULL,
+    description TEXT,
+    license TEXT,
+    published_at TIMESTAMP DEFAULT NULL,
+    dependencies TEXT[] DEFAULT NULL,
+    optional_dependencies TEXT[] DEFAULT NULL,
+    dev_dependencies TEXT[] DEFAULT NULL,
+    peer_dependencies TEXT[] DEFAULT NULL
+);