diff --git a/.env b/.env index e69de29..b7c9c42 100644 --- a/.env +++ b/.env @@ -0,0 +1,7 @@ +POSTGRES_HOST=localhost +POSTGRES_PORT=5433 +POSTGRES_USER=user +POSTGRES_PASSWORD=password +POSTGRES_DB=postgres + +DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable \ No newline at end of file diff --git a/.env.example b/.env.example index e69de29..4b80ec0 100644 --- a/.env.example +++ b/.env.example @@ -0,0 +1,8 @@ +POSTGRES_HOST=localhost // hostname of the PostgreSQL server +POSTGRES_PORT=5433 // port number of the PostgreSQL server +POSTGRES_USER=user // username for the PostgreSQL database +POSTGRES_PASSWORD=password // password for the PostgreSQL database +POSTGRES_DB=postgres // name of the PostgreSQL database + +# Database configuration (2/2) - Remember to set sslmode for production +DB_URL=postgres://user:password@localhost:5433/postgres?sslmode=disable \ No newline at end of file diff --git a/.gitignore b/.gitignore index ca94bf8..70b2b9b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,5 @@ - -.PHONY: help - - - -help: - @echo "Available commands:" - @echo " make help - Show this help message" \ No newline at end of file +.env +bin +bin/* +tmp +tmp/* \ No newline at end of file diff --git a/Makefile b/Makefile index 9d82bfd..e6b0d7d 100644 --- a/Makefile +++ b/Makefile @@ -16,11 +16,36 @@ dropBin: replicator-service: air -c configs/air/replicator-service.toml +.PHONY: migrate-up +migrate-up: + migrate -path migrations -database $(DB_URL) up + +.PHONY: migrate-down +migrate-down: + migrate -path migrations -database $(DB_URL) down + +.PHONY: migrate-force +migrate-force: + migrate -path migrations -database $(DB_URL) force $(version) + +.PHONY: complete-testing-npm-db +complete-testing-npm-db: + bash scripts/completeNpmDb.sh + +.PHONY: create-testing-npm-db +create-testing-npm-db: + bash scripts/createTestingNpmDb.sh + .PHONY: help help: @echo "Available commands:" - @echo "make help - Show this help message" - @echo "make init - Initialize pre-push hook" - @echo "make build - Build the project" - @echo "make dropBin - Drop the binary files" - @echo "make replicator-service - Run the replicator service with air in development mode" \ No newline at end of file + @echo "make help - Show this help message" + @echo "make init - Initialize pre-push hook" + @echo "make build - Build the project" + @echo "make dropBin - Drop the binary files" + @echo "make replicator-service - Run the replicator service with air in development mode" + @echo "make migrate-up - Run database migrations up" + @echo "make migrate-down - Run database migrations down" + @echo "make migrate-force - Force database migration to a specific version (usage: make migrate-force version=)" + @echo "make complete-testing-npm-db - Complete testing npm database" + @echo "make create-testing-npm-db - Create testing npm database" \ No newline at end of file diff --git a/README.md b/README.md index f7e447a..cf9c35c 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ API for tracking vulnerability occurrence frequencies. ```bash make init -2. Install Go ``` +2. Install Go This step depends on your operating system. @@ -51,3 +51,17 @@ go install github.com/air-verse/air@latest ```bash make replicator-service ``` + +4. pick up Postgres db + +- podman + +```bash +podman compose up -d +``` + + - or docker + +```bash +docker compose up -d +``` diff --git a/batch.json b/batch.json new file mode 100644 index 0000000..e69de29 diff --git a/bin/replicator-service.exe b/bin/replicator-service.exe index e3f03e4..54c3e37 100755 Binary files a/bin/replicator-service.exe and b/bin/replicator-service.exe differ diff --git a/cmd/replicator-service/main.go b/cmd/replicator-service/main.go index 629268c..b149bef 100644 --- a/cmd/replicator-service/main.go +++ b/cmd/replicator-service/main.go @@ -1,5 +1,17 @@ package main +import ( + "fmt" + + "github.com/trustpkg/trustpkg-api/db" + "github.com/trustpkg/trustpkg-api/internal/npm" +) + func main() { - println("Hello, World!") -} \ No newline at end of file + db.ConnectDb() + + err := npm.Pipeline() + if err != nil { + fmt.Println("commit error: ", err) + } +} diff --git a/db/postgres.go b/db/postgres.go new file mode 100644 index 0000000..4b4586f --- /dev/null +++ b/db/postgres.go @@ -0,0 +1,65 @@ +package db + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + goDotEnv "github.com/joho/godotenv" +) + +type postgresConfig struct { + host string + port string + user string + password string + dbName string +} + +var Pool *pgxpool.Pool + +func ConnectDb() { + if err := goDotEnv.Load(".env"); err != nil { + log.Println(".env file not found, using system environment variables") + } + + dsn := strings.TrimSpace(os.Getenv("DB_URL")) + if dsn == "" { + var dbConfig = postgresConfig{ + host: os.Getenv("POSTGRES_HOST"), + port: os.Getenv("POSTGRES_PORT"), + user: os.Getenv("POSTGRES_USER"), + password: os.Getenv("POSTGRES_PASSWORD"), + dbName: os.Getenv("POSTGRES_DB"), + } + + if dbConfig.host == "" || dbConfig.port == "" || dbConfig.user == "" || dbConfig.dbName == "" { + fmt.Fprintln(os.Stderr, "Missing DB config. Set DB_URL or POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, POSTGRES_PASSWORD, POSTGRES_DB (copy .env.example to .env).") + os.Exit(1) + } + + dsn = fmt.Sprintf("postgres://%s:%s@%s:%s/%s?sslmode=disable", dbConfig.user, dbConfig.password, dbConfig.host, dbConfig.port, dbConfig.dbName) + } + + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to connect to database: %v\n", err) + os.Exit(1) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err = pool.Ping(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to ping database: %v\n", err) + os.Exit(1) + } + + Pool = pool + + log.Println("Connected to postgres database successfully") +} \ No newline at end of file diff --git a/docker-compose.couchDb.yml b/docker-compose.couchDb.yml new file mode 100644 index 0000000..5c1c7fd --- /dev/null +++ b/docker-compose.couchDb.yml @@ -0,0 +1,11 @@ +services: + couchdb: + image: docker.io/library/couchdb:latest + restart: always + ports: + - 3200:5984 + environment: + COUCHDB_USER: admin + COUCHDB_PASSWORD: password + TZ: UTC + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a91647f..3af2cde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: - 8081:8080 depends_on: - db - network: + networks: - trustpkg-net volumes: diff --git a/go.mod b/go.mod index d616f02..2a1be88 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,28 @@ -module github.com/TymekGluch/trustpkg-api +module github.com/trustpkg/trustpkg-api go 1.26.5 + +require ( + github.com/jackc/pgx/v5 v5.10.0 + github.com/joho/godotenv v1.5.1 +) + +require ( + github.com/ebitengine/purego v0.10.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + golang.org/x/sys v0.41.0 // indirect +) + +require ( + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/shirou/gopsutil/v4 v4.26.7 + golang.org/x/sync v0.17.0 // indirect + golang.org/x/text v0.29.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c3ef935 --- /dev/null +++ b/go.sum @@ -0,0 +1,50 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE= +github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc= +github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/adaptive-worker/constants.go b/internal/adaptive-worker/constants.go new file mode 100644 index 0000000..8a0857a --- /dev/null +++ b/internal/adaptive-worker/constants.go @@ -0,0 +1,25 @@ +package adaptiveWorker + +const ( + minWorkers = 2 + maxWorkersPerCpu = 16 + + maxWorkerHeadroom = 0.2 + + maxCpuUsage = 0.6 + maxMemoryUsage = 0.8 + + timeFromRequestWorker = 30 + + skipCount = 2 + + badUsagePoints = 0 + RegularUsagePoints = 1 + GoodUsagePoints = 2 + ExcellentUsagePoints = 3 + + usageStateBad string = "bad" + usageStateRegular string = "regular" + usageStateGood string = "good" + usageStateExcellent string = "excellent" +) diff --git a/internal/adaptive-worker/controler.go b/internal/adaptive-worker/controler.go new file mode 100644 index 0000000..69a9017 --- /dev/null +++ b/internal/adaptive-worker/controler.go @@ -0,0 +1,69 @@ +package adaptiveWorker + +type Controler struct { + simultaneousWorkers int +} + +func (controler *Controler) Adjust(count int) { + controler.simultaneousWorkers = count +} + +func (controler *Controler) CalculateWorkers() error { + resourceUsage, err := CheckResourceUsage() + if err != nil { + controler.Adjust(minWorkers) + + return err + } + + maxSimultaneousWorkers := getMaxWorkers(resourceUsage.cpuCount) + maxSimultaneousWorkersHeadroom := getMaxWorkersHeadroom(resourceUsage.cpuCount) + + calculedData := calculedResourcesUsage{ + loadPercent: resourceUsage.loadPercent, + cpuPercent: resourceUsage.cpuPercent, + ramPercent: resourceUsage.ramPercent, + } + + isAtBoundary := controler.simultaneousWorkers >= maxSimultaneousWorkers + canEnterHeadroom := isAtBoundary && controler.simultaneousWorkers+skipCount <= maxSimultaneousWorkersHeadroom + + var simultaneousWorkers int + if controler.simultaneousWorkers == 0 { + simultaneousWorkers = minWorkers + } else { + simultaneousWorkers = controler.simultaneousWorkers + } + + if getResourceUsageState(calculedData) == usageStateExcellent { + if isAtBoundary && canEnterHeadroom { + controler.Adjust(simultaneousWorkers + skipCount) + } else { + controler.Adjust(simultaneousWorkers) + } + + return nil + } + + if getResourceUsageState(calculedData) == usageStateGood { + controler.Adjust(simultaneousWorkers + skipCount) + + return nil + } + + if getResourceUsageState(calculedData) == usageStateRegular { + controler.Adjust(simultaneousWorkers) + + return nil + } + + if getResourceUsageState(calculedData) == usageStateBad { + controler.Adjust(simultaneousWorkers - skipCount) + + return nil + } + + controler.Adjust(minWorkers) + + return nil +} diff --git a/internal/adaptive-worker/helpers.go b/internal/adaptive-worker/helpers.go new file mode 100644 index 0000000..2771cb8 --- /dev/null +++ b/internal/adaptive-worker/helpers.go @@ -0,0 +1,82 @@ +package adaptiveWorker + +import ( + "time" + + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/load" + "github.com/shirou/gopsutil/v4/mem" +) + +func CheckResourceUsage() (*resourcesUsage, error) { + cpuPercent, err := cpu.Percent(time.Second, false) + if err != nil { + return nil, err + } + + cpuCount, err := cpu.Counts(true) + + loadAvg, err := load.Avg() + if err != nil { + return nil, err + } + + memory, err := mem.VirtualMemory() + if err != nil { + return nil, err + } + + loadPercent := loadAvg.Load1 / float64(cpuCount) * 100 + + return &resourcesUsage{ + loadPercent: loadPercent, + loadAvg: loadAvg.Load1, + cpuPercent: cpuPercent[0], + ramPercent: memory.UsedPercent, + cpuCount: cpuCount, + }, nil +} + +func getMaxWorkers(cpuCount int) int { + return cpuCount * maxWorkersPerCpu +} + +func getMaxWorkersHeadroom(cpuCount int) int { + maxWorkersCount := getMaxWorkers(cpuCount) + + return maxWorkersCount + int(float64(maxWorkersCount)*maxWorkerHeadroom) +} + +func getPointsByUsage(value float64) int { + switch { + case value < 40: + return ExcellentUsagePoints + case value < 60: + return GoodUsagePoints + case value < 80: + return RegularUsagePoints + default: + return badUsagePoints + } +} + +func getResourceUsageState(resources calculedResourcesUsage) string { + sum := getPointsByUsage(resources.cpuPercent) + + getPointsByUsage(resources.loadPercent) + + getPointsByUsage(resources.ramPercent) + + if resources.ramPercent > 90 { + return usageStateBad + } + + switch { + case sum >= 8: + return usageStateExcellent + case sum >= 5: + return usageStateGood + case sum >= 2: + return usageStateRegular + default: + return usageStateBad + } +} diff --git a/internal/adaptive-worker/job.go b/internal/adaptive-worker/job.go new file mode 100644 index 0000000..0a1c9ca --- /dev/null +++ b/internal/adaptive-worker/job.go @@ -0,0 +1 @@ +package adaptiveWorker diff --git a/internal/adaptive-worker/models.go b/internal/adaptive-worker/models.go new file mode 100644 index 0000000..d22d809 --- /dev/null +++ b/internal/adaptive-worker/models.go @@ -0,0 +1,15 @@ +package adaptiveWorker + +type resourcesUsage struct { + loadPercent float64 + loadAvg float64 + cpuPercent float64 + ramPercent float64 + cpuCount int +} + +type calculedResourcesUsage struct { + loadPercent float64 + cpuPercent float64 + ramPercent float64 +} diff --git a/internal/npm/constants.go b/internal/npm/constants.go new file mode 100644 index 0000000..1917cab --- /dev/null +++ b/internal/npm/constants.go @@ -0,0 +1,6 @@ +package npm + +const ( + npmReplicationUrl = "https://replicate.npmjs.com/registry/_changes" + limit = "10000" +) diff --git a/internal/npm/helpers.go b/internal/npm/helpers.go new file mode 100644 index 0000000..0d0a19b --- /dev/null +++ b/internal/npm/helpers.go @@ -0,0 +1,17 @@ +package npm + +func getUniquePackages(packages []npmChange) []npmChange { + unique := make(map[string]npmChange, len(packages)) + + for _, item := range packages { + unique[item.ID] = item + } + + result := make([]npmChange, len(unique)) + + for _, item := range unique { + result = append(result, item) + } + + return result +} diff --git a/internal/npm/http.go b/internal/npm/http.go new file mode 100644 index 0000000..e6c28d9 --- /dev/null +++ b/internal/npm/http.go @@ -0,0 +1,46 @@ +package npm + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" +) + +func fetchNpmPackagesByCurrentSequence(ctx context.Context) (*NpmChangesResponse, error) { + client := &http.Client{} + + var sequence = 0 + + sequenceFromDb, err := selectLastSequence(ctx) + if err != nil { + panic(err) + } + if sequenceFromDb != 0 { + sequence = sequenceFromDb + } + fmt.Println("current sequecne: ", sequence) + + url := npmReplicationUrl + "?since=" + strconv.Itoa(sequence) + "&limit=" + limit + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var packagesData NpmChangesResponse + + err = json.NewDecoder(response.Body).Decode(&packagesData) + if err != nil { + return nil, err + } + + return &packagesData, nil +} diff --git a/internal/npm/models.go b/internal/npm/models.go new file mode 100644 index 0000000..62a3130 --- /dev/null +++ b/internal/npm/models.go @@ -0,0 +1,25 @@ +package npm + +type npmRevision struct { + Rev string `json:"rev"` +} + +type npmChange struct { + Seq int64 `json:"seq"` + ID string `json:"id"` + Deleted bool `json:"deleted"` + Changes []npmRevision `json:"changes"` +} + +type NpmChangesResponse struct { + LastSeq int `json:"last_seq"` + Results []npmChange `json:"results"` +} + +type dbDropPackagesBatchPayload struct { + packages []string +} + +type dbInsertPackagesBatchPayload struct { + packages []string +} diff --git a/internal/npm/replicator.go b/internal/npm/replicator.go new file mode 100644 index 0000000..c2105a2 --- /dev/null +++ b/internal/npm/replicator.go @@ -0,0 +1,2 @@ +package npm + diff --git a/internal/npm/repository.go b/internal/npm/repository.go new file mode 100644 index 0000000..9837827 --- /dev/null +++ b/internal/npm/repository.go @@ -0,0 +1,58 @@ +package npm + +import ( + "context" + "errors" + + _ "embed" + + "github.com/jackc/pgx/v5" + "github.com/trustpkg/trustpkg-api/db" +) + +var ( + //go:embed sql/delete_packages_batch.sql + queryDeletePackagesBatch string + + //go:embed sql/insert_last_sequence.sql + queryInsertLastSequence string + + //go:embed sql/insert_packages_batch.sql + queryInsertPackagesBatch string + + //go:embed sql/select_last_sequence.sql + querySelectLastSequence string +) + +func insertLastSequence(ctx context.Context, transaction pgx.Tx, sequence int) error { + _, err := transaction.Exec(ctx, queryInsertLastSequence, sequence) + + return err +} + +func selectLastSequence(ctx context.Context) (int, error) { + var sequence int + + err := db.Pool.QueryRow(ctx, querySelectLastSequence).Scan(&sequence) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return 0, nil + } + + return 0, err + } + + return sequence, nil +} + +func dropPackagesBatch(ctx context.Context, transaction pgx.Tx, payload dbDropPackagesBatchPayload) error { + _, err := transaction.Exec(ctx, queryDeletePackagesBatch, payload.packages) + + return err +} + +func insertPackagesBatch(ctx context.Context, transaction pgx.Tx, payload dbInsertPackagesBatchPayload) error { + _, err := transaction.Exec(ctx, queryInsertPackagesBatch, payload.packages) + + return err +} diff --git a/internal/npm/service.go b/internal/npm/service.go new file mode 100644 index 0000000..a8833bf --- /dev/null +++ b/internal/npm/service.go @@ -0,0 +1,64 @@ +package npm + +import ( + "context" + "fmt" + + "github.com/trustpkg/trustpkg-api/db" + adaptiveWorker "github.com/trustpkg/trustpkg-api/internal/adaptive-worker" +) + +func Pipeline() error { + ctx := context.Background() + + response, err := fetchNpmPackagesByCurrentSequence(ctx) + if err != nil { + return err + } + + unique := getUniquePackages(response.Results) + + var toAdd []string + var toRemove []string + + for _, uniquePackage := range unique { + + if uniquePackage.Deleted { + toRemove = append(toRemove, uniquePackage.ID) + } else { + toAdd = append(toAdd, uniquePackage.ID) + } + } + + transaction, err := db.Pool.Begin(ctx) + if err != nil { + return err + } + defer transaction.Rollback(ctx) + + if len(toAdd) > 0 { + err := insertPackagesBatch(ctx, transaction, dbInsertPackagesBatchPayload{packages: toAdd}) + if err != nil { + return err + } + } + + if len(toRemove) > 0 { + err := dropPackagesBatch(ctx, transaction, dbDropPackagesBatchPayload{packages: toRemove}) + if err != nil { + return err + } + } + + fmt.Println("overrite Sequence: ", response.LastSeq) + + err = insertLastSequence(ctx, transaction, response.LastSeq) + if err != nil { + return err + } + + data, _ := adaptiveWorker.CheckResourceUsage() + fmt.Println("data", data) + + return transaction.Commit(ctx) +} diff --git a/internal/npm/sql/delete_packages_batch.sql b/internal/npm/sql/delete_packages_batch.sql new file mode 100644 index 0000000..8acc4c3 --- /dev/null +++ b/internal/npm/sql/delete_packages_batch.sql @@ -0,0 +1,2 @@ +DELETE FROM packages +WHERE name = ANY($1::text[]); \ No newline at end of file diff --git a/internal/npm/sql/insert_last_sequence.sql b/internal/npm/sql/insert_last_sequence.sql new file mode 100644 index 0000000..15eddd9 --- /dev/null +++ b/internal/npm/sql/insert_last_sequence.sql @@ -0,0 +1,4 @@ +INSERT INTO npm_replication_state (id, last_seq) +VALUES (1, $1) +ON CONFLICT (id) +DO UPDATE SET last_seq = EXCLUDED.last_seq; \ No newline at end of file diff --git a/internal/npm/sql/insert_packages_batch.sql b/internal/npm/sql/insert_packages_batch.sql new file mode 100644 index 0000000..fc96bf7 --- /dev/null +++ b/internal/npm/sql/insert_packages_batch.sql @@ -0,0 +1,3 @@ +INSERT INTO packages (name) +SELECT unnest($1::text[]) +ON CONFLICT (name) DO NOTHING; \ No newline at end of file diff --git a/internal/npm/sql/select_last_sequence.sql b/internal/npm/sql/select_last_sequence.sql new file mode 100644 index 0000000..84a3d89 --- /dev/null +++ b/internal/npm/sql/select_last_sequence.sql @@ -0,0 +1,4 @@ +SELECT last_seq +FROM npm_replication_state +ORDER BY last_seq +DESC LIMIT 1; \ No newline at end of file diff --git a/migrations/0001_create_packages_table.down.sql b/migrations/0001_create_packages_table.down.sql new file mode 100644 index 0000000..3923e38 --- /dev/null +++ b/migrations/0001_create_packages_table.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS packages; + +DROP INDEX IF EXISTS idx_packages_name; \ No newline at end of file diff --git a/migrations/0001_create_packages_table.up.sql b/migrations/0001_create_packages_table.up.sql new file mode 100644 index 0000000..cc4a05c --- /dev/null +++ b/migrations/0001_create_packages_table.up.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS packages ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL UNIQUE +); + +CREATE INDEX IF NOT EXISTS idx_packages_name + ON packages(LOWER(name)); \ No newline at end of file diff --git a/migrations/0002_create_vulnerabilities_table.down.sql b/migrations/0002_create_vulnerabilities_table.down.sql new file mode 100644 index 0000000..751e97e --- /dev/null +++ b/migrations/0002_create_vulnerabilities_table.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS vulnerabilities; +DROP TABLE IF EXISTS package_vulnerabilities \ No newline at end of file diff --git a/migrations/0002_create_vulnerabilities_table.up.sql b/migrations/0002_create_vulnerabilities_table.up.sql new file mode 100644 index 0000000..84ed513 --- /dev/null +++ b/migrations/0002_create_vulnerabilities_table.up.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS vulnerabilities ( + id BIGSERIAL PRIMARY KEY, + osv_id TEXT NOT NULL UNIQUE, + cve_id TEXT, + summary TEXT, + description TEXT, + severity TEXT, + cvss_score NUMERIC(3, 1), + cvss_vector TEXT, + affected_ranges TEXT[], + fixed_versions TEXT[], + published_at TIMESTAMP, + modified_at TIMESTAMP, + reference_urls TEXT[] +); + +CREATE TABLE IF NOT EXISTS package_vulnerabilities ( + package_id BIGINT NOT NULL + REFERENCES packages(id) ON DELETE CASCADE, + + vulnerability_id BIGINT NOT NULL + REFERENCES vulnerabilities(id) ON DELETE CASCADE, + + PRIMARY KEY (package_id, vulnerability_id) +); \ No newline at end of file diff --git a/migrations/0003_create_npm_replication_state_table.down.sql b/migrations/0003_create_npm_replication_state_table.down.sql new file mode 100644 index 0000000..a190c9c --- /dev/null +++ b/migrations/0003_create_npm_replication_state_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS npm_replication_state; \ No newline at end of file diff --git a/migrations/0003_create_npm_replication_state_table.up.sql b/migrations/0003_create_npm_replication_state_table.up.sql new file mode 100644 index 0000000..1919c1a --- /dev/null +++ b/migrations/0003_create_npm_replication_state_table.up.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS npm_replication_state ( + id SMALLINT PRIMARY KEY DEFAULT 1, + last_seq BIGINT NOT NULL DEFAULT 0 +); \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh index 45ad77c..af49513 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -2,12 +2,14 @@ set -eu +source ./scripts/helpers/runLog.sh "Starting build script" + if [ ! -d "bin" ]; then mkdir bin fi if [ ! -d "cmd" ]; then - echo "cmd directory is missing" + source ./scripts/helpers/errorLog.sh "cmd directory is missing" exit 1 else for dir in cmd/*/; do @@ -18,7 +20,9 @@ else go build -o "bin/$dirname" "$mainFile" echo "binary $dirname created in bin directory" else - echo "main file $mainFile not found, skipping" + source ./scripts/helpers/errorLog.sh "main file $mainFile not found, skipping" fi done -fi \ No newline at end of file +fi + +source ./scripts/helpers/doneLog.sh "Build script completed successfully" \ No newline at end of file diff --git a/scripts/completeNpmDb.sh b/scripts/completeNpmDb.sh new file mode 100644 index 0000000..5be6a9c --- /dev/null +++ b/scripts/completeNpmDb.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash + +set -u + +COUCH_URL="http://localhost:3200" +COUCH_DB="npm" +COUCH_USER="admin" +COUCH_PASSWORD="password" + +NPM_REGISTRY="https://registry.npmjs.org" + +MAX_PARALLEL=5 +MAX_RETRIES=3 + +source ./scripts/helpers/runLog.sh "Starting completeNpmDb script" + +PACKAGES=( + express + lodash + react + axios + typescript + fastify + next + vite + eslint + prettier +) + +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} + +trap cleanup EXIT + + +fetch_package() { + local package="$1" + local output="$TMP_DIR/$package.json" + + echo "Downloading: $package" + + local attempt=1 + + while [ "$attempt" -le "$MAX_RETRIES" ]; do + + if curl \ + --fail \ + --silent \ + --show-error \ + --max-time 30 \ + "$NPM_REGISTRY/$package" \ + -o "$output"; then + + break + fi + + echo "WARNING: failed to download $package (attempt $attempt/$MAX_RETRIES)" >&2 + + rm -f "$output" + + if [ "$attempt" -eq "$MAX_RETRIES" ]; then + echo "ERROR: skipping $package" >&2 + return 1 + fi + + sleep 2 + + attempt=$((attempt + 1)) + done + + + if ! jq empty "$output" >/dev/null 2>&1; then + echo "ERROR: invalid JSON for $package" >&2 + rm -f "$output" + return 1 + fi + + + jq \ + --arg id "$package" \ + ' + ._id = $id | + .source = "npm" + ' \ + "$output" > "$output.tmp" + + mv "$output.tmp" "$output" + + echo "OK: $package" +} + + +upload_package() { + local package="$1" + local file="$TMP_DIR/$package.json" + + if [ ! -f "$file" ]; then + return 0 + fi + + echo "Uploading: $package" + + local response + + response=$(curl \ + --fail \ + --silent \ + --show-error \ + -u "$COUCH_USER:$COUCH_PASSWORD" \ + -H "Content-Type: application/json" \ + -X PUT \ + "$COUCH_URL/$COUCH_DB/$package" \ + --data-binary "@$file" \ + 2>&1) + + if [ $? -ne 0 ]; then + echo "ERROR: failed to upload $package" >&2 + echo "$response" >&2 + return 1 + fi + + echo "SAVED: $package" +} + + +export TMP_DIR +export NPM_REGISTRY +export MAX_RETRIES + +export -f fetch_package + + +echo "Downloading packages..." + +printf '%s\n' "${PACKAGES[@]}" | + xargs -n1 -P"$MAX_PARALLEL" \ + bash -c 'fetch_package "$1"' _ + + +echo +echo "Downloading completed." +echo + + +echo "Uploading packages..." + +SUCCESS=0 +FAILED=0 + +for package in "${PACKAGES[@]}"; do + + if upload_package "$package"; then + SUCCESS=$((SUCCESS + 1)) + else + FAILED=$((FAILED + 1)) + fi + +done + + +source ./scripts/helpers/doneLog.sh "completeNpmDb script completed successfully" \ No newline at end of file diff --git a/scripts/createTestingNpmDb.sh b/scripts/createTestingNpmDb.sh new file mode 100644 index 0000000..fc9acd7 --- /dev/null +++ b/scripts/createTestingNpmDb.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +source ./scripts/helpers/runLog.sh "Starting createTestingNpmDb script" + +curl -u admin:password \ + -X PUT \ + http://localhost:3200/npm + +source ./scripts/helpers/doneLog.sh "createTestingNpmDb script completed successfully" \ No newline at end of file diff --git a/scripts/dropBinaries.sh b/scripts/dropBinaries.sh index a515d77..40d47fe 100644 --- a/scripts/dropBinaries.sh +++ b/scripts/dropBinaries.sh @@ -2,6 +2,8 @@ set -e +source ./scripts/helpers/runLog.sh "Starting dropBinaries script" + shopt -s nullglob filesInBinDir=(bin/*) shopt -u nullglob @@ -18,4 +20,4 @@ for file in "${filesInBinDir[@]}"; do echo "file $filename removed from bin directory" done -echo "all files removed from bin directory" \ No newline at end of file +source ./scripts/helpers/doneLog.sh "All files removed from bin directory" \ No newline at end of file diff --git a/scripts/init.sh b/scripts/init.sh index 25af186..9d3205b 100644 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -5,6 +5,4 @@ source scripts/helpers/runLog.sh "Initializing pre-push hook" cp ./scripts/prePush.sh .git/hooks/pre-push chmod +x .git/hooks/pre-push -echo "Pre-push hook installed successfully!" - source scripts/helpers/doneLog.sh "Initialization git hooks complete successfully" \ No newline at end of file diff --git a/scripts/postBumpVersion.sh b/scripts/postBumpVersion.sh deleted file mode 100644 index 73df2b0..0000000 --- a/scripts/postBumpVersion.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function check_version_in_file() { - local file="$1" - local version="$2" - - if [[ ! -f "$file" ]]; then - echo "File $file not found, skipping version check" - return 0 - fi - - if ! grep -q "$version" "$file"; then - echo "Version $version not found in $file" - return 1 - fi -} - -function set_version_in_file() { - local file="$1" - local version="$2" - - if [[ ! -f "$file" ]]; then - echo "File $file not found, skipping version set" - return 0 - fi - - sed -i -E "s/[0-9]+\.[0-9]+\.[0-9]+/$version/g" "$file" - - git add "$file" - git commit -m "chore: update version in $file to $version" --no-verify -} - -TAG=$(git describe --tags --abbrev=0) -if [[ $TAG == "" ]]; then - TAG="0.0.0" -fi - -files=( - "README.md" - "./cmd/morphixis-mail-service/main.go" -) - -for file in "${files[@]}"; do - if [[ ! -f "$file" ]]; then - echo "$file not found, skipping" - continue - fi - - if check_version_in_file "$file" "$TAG"; then - echo "Version $TAG found in $file, skipping" - else - echo "Version $TAG not found in $file, updating..." - set_version_in_file "$file" "$TAG" - fi -done - diff --git a/scripts/postmanSync.sh b/scripts/postmanSync.sh deleted file mode 100644 index 592671a..0000000 --- a/scripts/postmanSync.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if [[ -f ".env" ]]; then - set -a - source .env - set +a -fi - -if [[ -z "${POSTMAN_API_KEY:-}" ]]; then - echo "postman-sync: POSTMAN_API_KEY not set, skipping" - exit 0 -fi - -if [[ -z "${POSTMAN_COLLECTION_UID:-}" ]]; then - echo "postman-sync: POSTMAN_COLLECTION_UID not set, skipping" - exit 0 -fi - -if [[ ! "${POSTMAN_COLLECTION_UID}" =~ ^[a-zA-Z0-9_-]+-[a-zA-Z0-9_-]+$ ]]; then - echo "postman-sync: invalid POSTMAN_COLLECTION_UID format" - exit 1 -fi - -COLLECTION_FILE="${POSTMAN_COLLECTION_FILE:-postman/morphyxis-mail-service.postman_collection.json}" - -TMPFILE=$(mktemp --suffix=.json) -trap 'rm -f "$TMPFILE"' EXIT - -if [[ ! -f "$COLLECTION_FILE" ]]; then - echo "postman-sync: local file not found, pulling from Postman..." - - mkdir -p "$(dirname "$COLLECTION_FILE")" - - RESPONSE=$(curl -s -w "\n%{http_code}" \ - "https://api.getpostman.com/collections/${POSTMAN_COLLECTION_UID}" \ - -H "x-api-key: ${POSTMAN_API_KEY}") - - HTTP_CODE=$(echo "$RESPONSE" | tail -n1) - BODY=$(echo "$RESPONSE" | head -n-1) - - if [[ "$HTTP_CODE" != "200" ]]; then - echo "postman-sync: pull failed (HTTP $HTTP_CODE): $BODY" - exit 1 - fi - - echo "$BODY" | grep -o '"collection":{.*' | sed 's/"collection"://' | sed 's/}$//' > "$TMPFILE" - - if command -v jq &>/dev/null; then - echo "$BODY" | jq '.collection' > "$COLLECTION_FILE" - else - echo "$BODY" > "$COLLECTION_FILE" - fi - - echo "postman-sync: saved to $COLLECTION_FILE" - exit 0 -fi - -echo "postman-sync: pushing to Postman (uid: ${POSTMAN_COLLECTION_UID})..." - -FIRST_KEY=$(head -c 20 "$COLLECTION_FILE") -if [[ "$FIRST_KEY" == *'"collection"'* ]]; then - cp "$COLLECTION_FILE" "$TMPFILE" -else - printf '{"collection":' > "$TMPFILE" - cat "$COLLECTION_FILE" >> "$TMPFILE" - printf '}' >> "$TMPFILE" -fi - -RESPONSE=$(curl -s -w "\n%{http_code}" \ - -X PUT \ - "https://api.getpostman.com/collections/${POSTMAN_COLLECTION_UID}" \ - -H "x-api-key: ${POSTMAN_API_KEY}" \ - -H "Content-Type: application/json" \ - --data-binary "@${TMPFILE}") - -HTTP_CODE=$(echo "$RESPONSE" | tail -n1) -BODY=$(echo "$RESPONSE" | head -n-1) - -if [[ "$HTTP_CODE" != "200" ]]; then - echo "postman-sync: push failed (HTTP $HTTP_CODE): $BODY" - exit 1 -fi - -echo "postman-sync: done" diff --git a/scripts/prePush.sh b/scripts/prePush.sh index 9264e5c..33f3a91 100644 --- a/scripts/prePush.sh +++ b/scripts/prePush.sh @@ -2,5 +2,8 @@ set -e +source ./scripts/helpers/runLog.sh "Starting pre-push script" + source ./scripts/bumpVersion.sh -source ./scripts/postBumpVersion.sh + +source ./scripts/helpers/doneLog.sh "Pre-push script completed successfully" \ No newline at end of file