From 1c0ff2b104da68d38d24c7b654b6342436fa984e Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 21 Aug 2025 13:49:57 +0200 Subject: [PATCH 001/184] Create new function for ingest message and correlation id --- sda/cmd/api/api.go | 54 ++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b0bfe2255..d22595ad6 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -317,8 +317,37 @@ func getFiles(c *gin.Context) { c.JSON(200, files) } +// ingestFile function sends the ingest message +// to the broker func ingestFile(c *gin.Context) { + ingest, corrID, err := msgInfoFilePath(c) + if err != nil { + return + } + + marshaledMsg, _ := json.Marshal(&ingest) + if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + + return + } + + err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "ingest", marshaledMsg) + if err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + c.Status(http.StatusOK) +} + +// ingestMsgFilePath parses the JSON payload from the request to construct an ingestion trigger message. +// It validates the payload and retrieves the correlation ID for the file using the user and file path. +// Returns the parsed ingestion trigger struct, the correlation ID, and an error if any occurred. +func ingestMsgFilePath(c *gin.Context) (schema.IngestionTrigger, string, error) { var ingest schema.IngestionTrigger + // Bind ingest and payload if err := c.BindJSON(&ingest); err != nil { c.AbortWithStatusJSON( http.StatusBadRequest, @@ -328,17 +357,9 @@ func ingestFile(c *gin.Context) { }, ) - return - } - - ingest.Type = "ingest" - marshaledMsg, _ := json.Marshal(&ingest) - if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - - return + return schema.IngestionTrigger{}, "", err } - + // Find the correlation id of the file corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") if err != nil { if corrID == "" { @@ -347,17 +368,12 @@ func ingestFile(c *gin.Context) { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } - return - } - - err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "ingest", marshaledMsg) - if err != nil { - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return + return schema.IngestionTrigger{}, "", err } + // Add type in message payload + ingest.Type = "ingest" - c.Status(http.StatusOK) + return ingest, corrID, nil } // The deleteFile function deletes files from the inbox and marks them as From 92022fbee4b23288a0e3ef1171fc28506cb4e19e Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 21 Aug 2025 15:31:25 +0200 Subject: [PATCH 002/184] Create function for retrieving user and path --- sda/internal/database/database.go | 5 +++++ sda/internal/database/db_functions.go | 31 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index bb00a755f..f02593e2f 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -61,6 +61,11 @@ type DatasetInfo struct { Timestamp string `json:"timeStamp"` } +type IngestInfo struct { + User string + InboxPath string +} + // SchemaName is the name of the remote database schema to query var SchemaName = "sda" diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 40372c8d9..e11dbf870 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1174,3 +1174,34 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { return accessions, nil } + +// GetUserAndPathFromUUID() retrieves user and path by giving the file UUID +func (dbs *SDAdb) GetUserAndPathFromUUID(fileUUID string) (IngestInfo, error) { + var ( + i IngestInfo + err error + ) + + for count := 0; count <= RetryTimes; count++ { + i, err = dbs.getUserAndPathFromUUID(fileUUID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return i, err +} + +// getUserAndPathFromUUID() is the actual function performing work for GetUserAndPathFromUUID +func (dbs *SDAdb) getUserAndPathFromUUID(fileUUID string) (IngestInfo, error) { + dbs.checkAndReconnectIfNeeded() + + const query = "SELECT submission_user, submission_file_path from sda.files WHERE id = $1;" + var info IngestInfo + if err := dbs.DB.QueryRow(query, fileUUID).Scan(&info.User, &info.InboxPath); err != nil { + return IngestInfo{}, err + } + + return info, nil +} From 9384310149dd5f1691770972460249692496e656 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 21 Aug 2025 15:32:05 +0200 Subject: [PATCH 003/184] Tests for the db function --- sda/internal/database/db_functions_test.go | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index bbdbc7357..331957a68 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1285,3 +1285,33 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileID2) } + +func (suite *DatabaseTests) TestGetUserAndPathFromUUID_Found() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "failed to create new connection") + + // Register a file to get a valid UUID + filePath := "/dummy_user.org/Dummy_folder/dummyfile.c4gh" + user := "dummy@user.org" + fileID, err := db.RegisterFile(filePath, user) + assert.NoError(suite.T(), err, "failed to register file in database") + + info, err := db.GetUserAndPathFromUUID(fileID) + assert.NoError(suite.T(), err, "failed to get user and path from UUID") + assert.Equal(suite.T(), user, info.User) + assert.Equal(suite.T(), filePath, info.InboxPath) + db.Close() +} + +func (suite *DatabaseTests) TestGetUserAndPathFromUUID_NotFound() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "failed to create new connection") + + // Use a non-existent UUID + invalidUUID := "abc-123" + info, err := db.GetUserAndPathFromUUID(invalidUUID) + assert.Error(suite.T(), err, "expected error for non-existent UUID") + assert.Empty(suite.T(), info.User) + assert.Empty(suite.T(), info.InboxPath) + db.Close() +} From b37b82780df8942c269e85848508e9dcb24e8dbe Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 22 Aug 2025 11:18:56 +0200 Subject: [PATCH 004/184] Create function in api for accepting file id as parameter --- .github/integration/sda/rbac.json | 4 +-- sda/cmd/api/api.go | 51 +++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 515d7bb45..437ea2cf9 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -22,7 +22,7 @@ }, { "role": "submission", - "path": "/file/ingest", + "path": "/file/ingest*", "action": "POST" }, { @@ -74,4 +74,4 @@ "rolebinding": "admin" } ] - } \ No newline at end of file + } diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index d22595ad6..44db5518d 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -320,9 +320,22 @@ func getFiles(c *gin.Context) { // ingestFile function sends the ingest message // to the broker func ingestFile(c *gin.Context) { - ingest, corrID, err := msgInfoFilePath(c) - if err != nil { - return + var ( + ingest schema.IngestionTrigger + corrID string + err error + ) + fileID := c.Query("fileid") + if fileID == "" { + ingest, corrID, err = ingestMsgFilePath(c) + if err != nil { + return + } + } else { + ingest, corrID, err = ingestMsgFileID(c, fileID) + if err != nil { + return + } } marshaledMsg, _ := json.Marshal(&ingest) @@ -376,6 +389,38 @@ func ingestMsgFilePath(c *gin.Context) (schema.IngestionTrigger, string, error) return ingest, corrID, nil } +// ingestMsgFileID constructs an ingestion trigger message using a file UUID provided as a query parameter. +// It checks that no JSON payload is present in the request; if both are provided, it returns a 400 Bad Request. +// The function retrieves the user and file path from the database using the file UUID. +// If the file UUID is not found, it responds with a 404 Not Found error. +// Returns the constructed ingestion trigger struct, the correlation ID (file UUID), and an error if any occurred. +func ingestMsgFileID(c *gin.Context, fileUUID string) (schema.IngestionTrigger, string, error) { + var ingest schema.IngestionTrigger + // Check if payload is provided as well + if c.Request.ContentLength > 0 { + c.AbortWithStatusJSON(http.StatusBadRequest, "Both file ID parameter and payload provided. Choose one") + + return schema.IngestionTrigger{}, "", errors.New("add either parameter or payload, not both") + } + // Get the user and the inbox filepath + ingestInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) + if err != nil { + c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") + + return schema.IngestionTrigger{}, "", err + } + + // Information needed for the ingest message + ingest.Type = "ingest" + ingest.User = ingestInfo.User + ingest.FilePath = ingestInfo.InboxPath + // For BP the file UUID and the correlation ID are the same. + // TODO: If in GDI they are not the same then change the line below + corrID := fileUUID + + return ingest, corrID, nil +} + // The deleteFile function deletes files from the inbox and marks them as // discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { From 0bb5ea8bfb0a96d404ec4c376bd114ffc4fbcd67 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 22 Aug 2025 15:27:08 +0200 Subject: [PATCH 005/184] Unit tests for the msgInfoFileID function --- sda/cmd/api/api_test.go | 70 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 85f57fb79..3afe49c7f 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -444,7 +444,7 @@ func (s *TestSuite) SetupSuite() { s.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, - {"role":"submission","path":"/file/ingest","action":"POST"}, + {"role":"submission","path":"/file/ingest*","action":"POST"}, {"role":"submission","path":"/file/accession","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, @@ -1197,6 +1197,74 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { assert.Contains(s.T(), string(b), "sql: no rows in result set") } +func (s *TestSuite) TestIngestMsgFileID() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("POST", "/file/ingest", nil) + + ingest, corrID, err := ingestMsgFileID(c, fileID) + assert.NoError(s.T(), err) + assert.Equal(s.T(), user, ingest.User) + assert.Equal(s.T(), filePath, ingest.FilePath) + assert.Equal(s.T(), fileID, corrID) +} + +func (s *TestSuite) TestIngestMsgFileID_NotFound() { + user := "dummy" + filePath := "/inbox/dummy/file10.c4gh" + + _, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("POST", "/file/ingest", nil) + + ingest, corrID, err := ingestMsgFileID(c, "random-id") + assert.Error(s.T(), err) + assert.Contains(s.T(), w.Body.String(), "file ID not found") + assert.Equal(s.T(), http.StatusNotFound, w.Code) + assert.Empty(s.T(), ingest) + assert.Empty(s.T(), corrID) +} + +func (s *TestSuite) TestIngestMsgFileID_PayloadProvided() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + }) + c.Request = httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(payload)) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + ingest, corrID, err := ingestMsgFileID(c, fileID) + assert.Error(s.T(), err) + assert.Contains(s.T(), w.Body.String(), "Both file ID parameter and payload provided. Choose one") + assert.Equal(s.T(), http.StatusBadRequest, w.Code) + assert.Empty(s.T(), ingest) + assert.Empty(s.T(), corrID) +} + func (s *TestSuite) TestSetAccession() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" From 57665afb26c01b2c0771e02c088bccb372219670 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 25 Aug 2025 15:11:09 +0200 Subject: [PATCH 006/184] Modify unit tests for ingestFile function --- sda/cmd/api/api_test.go | 118 +++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 62 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 3afe49c7f..52269a61a 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1038,7 +1038,7 @@ func (s *TestSuite) TestRBAC_emptyPolicy() { assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) assert.Contains(s.T(), string(b), "not authorized") } -func (s *TestSuite) TestIngestFile() { +func (s *TestSuite) TestIngestFile_WithPayload() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" m, err := model.NewModelFromString(jsonadapter.Model) @@ -1097,93 +1097,89 @@ func (s *TestSuite) TestIngestFile() { assert.Equal(s.T(), 1, data.MessagesReady) } -func (s *TestSuite) TestIngestFile_NoUser() { +func (s *TestSuite) TestIngestFile_WithFileID() { user := "dummy" - filePath := "/inbox/dummy/file10.c4gh" - + filePath := "/inbox/dummy/file11.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") + assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(s.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + assert.NoError(s.T(), err) - Conf.Broker.SchemasPath = "../../schemas/isolated" - - type ingest struct { - FilePath string `json:"filepath"` - User string `json:"user"` - } - ingestMsg, _ := json.Marshal(ingest{User: "", FilePath: filePath}) - // Mock request and response holders w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) + r := httptest.NewRequest("POST", "/file/ingest?fileid="+fileID, nil) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) - router.POST("/file/ingest", ingestFile) - + router.POST("/file/ingest", rbac(e), ingestFile) router.ServeHTTP(w, r) + okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + // verify that the message shows up in the queue + time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives + client := http.Client{Timeout: 5 * time.Second} + req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody) + req.SetBasicAuth("guest", "guest") + res, err := client.Do(req) + assert.NoError(s.T(), err, "failed to query broker") + var data struct { + MessagesReady int `json:"messages_ready"` + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + assert.NoError(s.T(), err, "failed to read response from broker") + err = json.Unmarshal(body, &data) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (s *TestSuite) TestIngestFile_WrongUser() { - user := "dummy" - filePath := "/inbox/dummy/file10.c4gh" +func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { + user := "dummy" + filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") + assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(s.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + assert.NoError(s.T(), err) - Conf.Broker.SchemasPath = "../../schemas/isolated" - - type ingest struct { - FilePath string `json:"filepath"` - User string `json:"user"` - } - ingestMsg, _ := json.Marshal(ingest{User: "foo", FilePath: filePath}) - // Mock request and response holders w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + }) + r := httptest.NewRequest("POST", "/file/ingest?fileid="+fileID, bytes.NewBuffer(payload)) + r.Header.Add("Authorization", "Bearer "+s.Token) + r.Header.Set("Content-Type", "application/json") _, router := gin.CreateTestContext(w) - router.POST("/file/ingest", ingestFile) - + router.POST("/file/ingest", rbac(e), ingestFile) router.ServeHTTP(w, r) - okResponse := w.Result() - defer okResponse.Body.Close() - b, _ := io.ReadAll(okResponse.Body) - assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(s.T(), string(b), "sql: no rows in result set") -} - -func (s *TestSuite) TestIngestFile_WrongFilePath() { - user := "dummy" - filePath := "/inbox/dummy/file10.c4gh" - - fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(s.T(), err, "failed to update satus of file in database") - - gin.SetMode(gin.ReleaseMode) - assert.NoError(s.T(), setupJwtAuth()) - Conf.Broker.SchemasPath = "../../schemas/isolated" + resp := w.Result() + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(body), "Both file ID parameter and payload provided") +} - type ingest struct { - FilePath string `json:"filepath"` - User string `json:"user"` - } - ingestMsg, _ := json.Marshal(ingest{User: "dummy", FilePath: "bad/path"}) - // Mock request and response holders +func (s *TestSuite) TestIngestFile_NoFileIDnoPayload() { w := httptest.NewRecorder() - r := httptest.NewRequest("POST", "/file/ingest", bytes.NewBuffer(ingestMsg)) + r := httptest.NewRequest("POST", "/file/ingest", nil) r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) @@ -1192,9 +1188,7 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - b, _ := io.ReadAll(okResponse.Body) assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(s.T(), string(b), "sql: no rows in result set") } func (s *TestSuite) TestIngestMsgFileID() { From 90986416d9e59952cccdab66044b96bfcab45dda Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 26 Aug 2025 22:45:15 +0200 Subject: [PATCH 007/184] Add test for ingesting file via file id --- .../tests/sda/60_api_admin_test.sh | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 2094287a1..ecec6ff81 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -216,4 +216,30 @@ if [ "$resp" != "404" ]; then exit 1 fi -echo "API admin tests completed successfully" \ No newline at end of file +# Test ingesting file by using the file id +echo "Ingest file by using file ID" +# Reupload a file under a different name +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ingest/NB12878-ingest.bam.c4gh +sleep 3 +# Find the file id of the uploaded file +new_fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/ingest/NB12878-ingest.bam.c4gh") | .fileID')" +# ingest the file +ingest_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/file/ingest?fileid=$new_fileid")" +if [ "$ingest_resp" != "200" ]; then + echo "Error when requesting to ingesting file by the use of file id, expected 200 got: $ingest_resp" + exit 1 +fi +# Check that the file is ingested and verified +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$new_fileid' order by started_at desc limit 1;")" = "verified" ]; do + echo "waiting for verified to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 10 ]; then + echo "::error::Time out while waiting for verified to complete" + exit 1 + fi + sleep 2 +done +echo "Ingestion by using file ID finished successfully" + +echo "API admin tests completed successfully" From 62c85721bb0b7ab9e2b047bcb33c8d29a7b9bd97 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 26 Aug 2025 23:25:58 +0200 Subject: [PATCH 008/184] Modify swagger --- sda/cmd/api/swagger_v1.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index 4904de349..dda46af7b 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -181,19 +181,33 @@ paths: description: Internal application error. /file/ingest: post: - description: Trigger ingestion of a given file. + description: | + Trigger ingestion of a given file. + You can provide either a JSON payload with `user` and `filepath`, or a `fileid` query parameter. + If both are provided, a 400 Bad Request is returned. + parameters: + - in: query + name: fileid + schema: + type: string + required: false + description: UUID of the file to ingest. If provided, payload must be empty. requestBody: content: application/json: schema: $ref: "#/components/schemas/FileIngest" + required: false responses: "200": description: Successful operation. "400": - description: Bad payload + description: | + Bad payload. Returned if both fileid and payload are provided, or if payload is invalid. "401": description: Authentication failure. + "404": + description: File ID not found. "500": description: Internal application error. /file/{userName}/{fileID}: From 9aed12dd9818e540f02a7713190ab25f01a391a3 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 28 Aug 2025 11:23:44 +0200 Subject: [PATCH 009/184] Update readme for file/ingest --- sda/cmd/api/api.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 1a124c813..2a9d2850b 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -49,21 +49,31 @@ Endpoints: Admin endpoints are only available to a set of whitelisted users specified in the application config. - `/file/ingest` - - accepts `POST` requests with JSON data with the format: `{"filepath": "", "user": ""}` + - accepts `POST` requests with either: + - JSON data: `{"filepath": "", "user": ""}` + - OR a `fileid` query parameter: `/file/ingest?fileid=` - triggers the ingestion of the file. + - If both a JSON payload and a `fileid` query parameter are provided in the same request, a `400 Bad Request` is returned. + - Error codes - - `200` Query execute ok. - - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination. + - `200` Query executed successfully. + - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and fileid provided, or invalid JSON). - `401` Token user is not in the list of admins. + - `404` File ID not found. - `500` Internal error due to DB or MQ failures. - Example: + Example (JSON payload): ```bash curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/ingest ``` + Example (fileid query parameter): + + ```bash + curl -H "Authorization: Bearer $token" -X POST "https://HOSTNAME/file/ingest?fileid=" + - `/file/accession` - accepts `POST` requests with JSON data with the format: `{"accession_id": "", "filepath": "", "user": ""}` - assigns accession ID to the file. From 425bc45a047255dff354cedd372d6d87e6ec753d Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 28 Aug 2025 11:31:48 +0200 Subject: [PATCH 010/184] Refactor setAccession function to extract JSON parsing and correlation ID retrieval into accessionMsgFilePath function --- sda/cmd/api/api.go | 49 ++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 44db5518d..6ea298c50 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -596,6 +596,32 @@ func downloadFile(c *gin.Context) { } func setAccession(c *gin.Context) { + accession, corrID, err := accessionMsgFilePath(c) + + marshaledMsg, _ := json.Marshal(&accession) + if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { + log.Debugln(err.Error()) + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + + return + } + + err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "accession", marshaledMsg) + if err != nil { + log.Debugln(err.Error()) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + c.Status(http.StatusOK) +} + +// accessionMsgFilePath parses the JSON payload from the request. +// It validates the payload and retrieves all the information needed +// to construct the accession payload message. +// Returns the accession payload, the correlation ID, and an error if any occurred. +func accessionMsgFilePath(c *gin.Context) (schema.IngestionAccession, string, error) { var accession schema.IngestionAccession if err := c.BindJSON(&accession); err != nil { c.AbortWithStatusJSON( @@ -606,7 +632,7 @@ func setAccession(c *gin.Context) { }, ) - return + return schema.IngestionAccession{}, "", err } corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") @@ -617,7 +643,7 @@ func setAccession(c *gin.Context) { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } - return + return schema.IngestionAccession{}, "", err } fileInfo, err := Conf.API.DB.GetFileInfo(corrID) @@ -625,28 +651,13 @@ func setAccession(c *gin.Context) { log.Debugln(err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - return + return schema.IngestionAccession{}, "", err } accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}} accession.Type = "accession" - marshaledMsg, _ := json.Marshal(&accession) - if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { - log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - - return - } - err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "accession", marshaledMsg) - if err != nil { - log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return - } - - c.Status(http.StatusOK) + return accession, corrID, nil } func createDataset(c *gin.Context) { From af49e8e99ff098b27135ac50e95e24df8e5ff162 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 28 Aug 2025 15:00:55 +0200 Subject: [PATCH 011/184] Replace db function for finding checksum with a simpler one --- sda/cmd/api/api.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 6ea298c50..2b66854ac 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -646,15 +646,18 @@ func accessionMsgFilePath(c *gin.Context) (schema.IngestionAccession, string, er return schema.IngestionAccession{}, "", err } - fileInfo, err := Conf.API.DB.GetFileInfo(corrID) + // For the BP case the correlation id is the same with the file id. + // TODO: If in GDi the IDs are different need to find the file id + fileUUID := corrID + fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileUUID) if err != nil { log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") return schema.IngestionAccession{}, "", err } - accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}} + accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} accession.Type = "accession" return accession, corrID, nil From 5b763f12540f0556061f23279ef5d250d7e1be1b Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 28 Aug 2025 15:30:39 +0200 Subject: [PATCH 012/184] Add accessionMsgFileID function which creates accession payload from endpoint parameters --- sda/cmd/api/api.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 2b66854ac..763354b1c 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -663,6 +663,47 @@ func accessionMsgFilePath(c *gin.Context) (schema.IngestionAccession, string, er return accession, corrID, nil } +// accessionMsgFileID constructs an accession message using a file UUID and accession ID provided as parameters. +// It checks that no JSON payload is present in the request; if both are provided, it returns a 400 Bad Request. +// The function retrieves the user, file path, and decrypted checksum from the database using the file UUID. +// If the file UUID or checksum is not found, it responds with a 404 Not Found error. +// Returns the accession payload, the correlation ID, and an error if any occurred. +func accessionMsgFileID(c *gin.Context, fileUUID, acccessionID string) (schema.IngestionAccession, string, error) { + var accession schema.IngestionAccession + // Check if payload is provided as well + if c.Request.ContentLength > 0 { + c.AbortWithStatusJSON(http.StatusBadRequest, "Both parameters and json payload provided. Choose one") + + return schema.IngestionAccession{}, "", errors.New("add either parameters or json payload, not both") + } + // Get the user and the inbox filepath + userPathInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) + if err != nil { + c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") + + return schema.IngestionAccession{}, "", err + } + // Get the decrypted checksum + fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileUUID) + if err != nil { + log.Debugln(err.Error()) + c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") + + return schema.IngestionAccession{}, "", err + } + // Information needed for the ingest message + accession.Type = "accession" + accession.User = userPathInfo.User + accession.FilePath = userPathInfo.InboxPath + accession.AccessionID = acccessionID + accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} + // For BP the file UUID and the correlation ID are the same. + // TODO: If in GDI they are not the same then change the line below + corrID := fileUUID + + return accession, corrID, nil +} + func createDataset(c *gin.Context) { var dataset dataset if err := c.BindJSON(&dataset); err != nil { From a886ed50d056d8d54602be980d5437242755d182 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 29 Aug 2025 15:54:28 +0200 Subject: [PATCH 013/184] Enhance setAccession function to handle query parameters and update rbac policy to allow wildcard path for accession --- .github/integration/sda/rbac.json | 2 +- sda/cmd/api/api.go | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 437ea2cf9..96de661b6 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -27,7 +27,7 @@ }, { "role": "submission", - "path": "/file/accession", + "path": "/file/accession*", "action": "POST" }, { diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 763354b1c..f485f7188 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -595,8 +595,30 @@ func downloadFile(c *gin.Context) { c.Status(http.StatusOK) } +// setAccession handles requests to assign an accession ID to a file. +// It accepts either a JSON payload with user and file path, or query parameters for fileid and accessionid. +// If both payload and parameters are provided, a 400 Bad Request is returned. +// The function validates the request, constructs the accession message, and sends it to the broker. +// Returns 200 OK on success, or an appropriate error code on failure. func setAccession(c *gin.Context) { - accession, corrID, err := accessionMsgFilePath(c) + var ( + accession schema.IngestionAccession + corrID string + err error + ) + accessionID := c.Query("accessionid") + fileID := c.Query("fileid") + if fileID == "" || accessionID == "" { + accession, corrID, err = accessionMsgFilePath(c) + if err != nil { + return + } + } else { + accession, corrID, err = accessionMsgFileID(c, fileID, accessionID) + if err != nil { + return + } + } marshaledMsg, _ := json.Marshal(&accession) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { From e1c4cc1b86ef3f2372bf27be179f43425a8d1ee9 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 1 Sep 2025 14:47:48 +0200 Subject: [PATCH 014/184] Add tests for accession message file path handling and create helper for verified test file --- sda/cmd/api/api_test.go | 110 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 52269a61a..fab1ef4e2 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "hash" "io" "net" "net/http" @@ -297,6 +298,35 @@ type TestSuite struct { GrpcListener GrpcListener } +func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, hash.Hash) { + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update status of file in database") + + encSha := sha256.New() + _, err = encSha.Write([]byte("Checksum")) + assert.NoError(s.T(), err) + + decSha := sha256.New() + _, err = decSha.Write([]byte("DecryptedChecksum")) + assert.NoError(s.T(), err) + + fileInfo := database.FileInfo{ + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), + Size: 1000, + Path: filePath, + DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), + DecryptedSize: 948, + } + err = Conf.API.DB.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") + err = Conf.API.DB.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Verified") + + return fileID, decSha +} + func (s *TestSuite) TestShutdown() { Conf = &config.Config{} Conf.Broker = broker.MQConf{ @@ -1413,6 +1443,86 @@ func (s *TestSuite) TestSetAccession_WrongFormat() { assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } +func (s *TestSuite) TestAccessionMsgFilePath() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + fileID, decSha := helperCreateVerifiedTestFile(s, user, filePath) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + "accession_id": accessionID, + }) + c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + accession, corrID, err := accessionMsgFilePath(c) + assert.NoError(s.T(), err) + assert.Equal(s.T(), user, accession.User) + assert.Equal(s.T(), filePath, accession.FilePath) + assert.Equal(s.T(), accessionID, accession.AccessionID) + assert.Equal(s.T(), fmt.Sprintf("%x", decSha.Sum(nil)), accession.DecryptedChecksums[0].Value) + assert.Equal(s.T(), fileID, corrID) +} + +func (s *TestSuite) TestAccessionMsgFilePath_WrongUser() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + payload, _ := json.Marshal(map[string]string{ + "user": "no-dummy-user", + "filepath": filePath, + "accession_id": accessionID, + }) + c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + accession, corrID, err := accessionMsgFilePath(c) + assert.Error(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, w.Code) + assert.Empty(s.T(), accession) + assert.Empty(s.T(), corrID) +} + +func (s *TestSuite) TestAccessionMsgFilePath_WrongPath() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": "random/folder/dumfile.c4gh", + "accession_id": accessionID, + }) + c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + accession, corrID, err := accessionMsgFilePath(c) + assert.Error(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, w.Code) + assert.Empty(s.T(), accession) + assert.Empty(s.T(), corrID) +} + func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" From 0819583a71f8e1af908c4201a03215fa5581269a Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 1 Sep 2025 15:11:04 +0200 Subject: [PATCH 015/184] Add tests for accessionMsgFileID function and handle invalid fileID scenario --- sda/cmd/api/api_test.go | 46 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index fab1ef4e2..39f2806aa 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1523,6 +1523,52 @@ func (s *TestSuite) TestAccessionMsgFilePath_WrongPath() { assert.Empty(s.T(), corrID) } +func (s *TestSuite) TestAccessionMsgFileID() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + + // Create and verify test file + fileID, decSha := helperCreateVerifiedTestFile(s, user, filePath) + + // Set up Gin context with no payload + gin.SetMode(gin.ReleaseMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+fileID+"&accessionid="+accessionID, nil) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + accession, corrID, err := accessionMsgFileID(c, fileID, accessionID) + assert.NoError(s.T(), err) + assert.Equal(s.T(), user, accession.User) + assert.Equal(s.T(), filePath, accession.FilePath) + assert.Equal(s.T(), accessionID, accession.AccessionID) + assert.Equal(s.T(), "accession", accession.Type) + assert.Equal(s.T(), fileID, corrID) + assert.Equal(s.T(), fmt.Sprintf("%x", decSha.Sum(nil)), accession.DecryptedChecksums[0].Value) +} + +func (s *TestSuite) TestAccessionMsgFileID_WrongFileID() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + // Register file and then use a random fileID + _, _ = helperCreateVerifiedTestFile(s, user, filePath) + randomFileID := "non-existent-file-id" + + gin.SetMode(gin.ReleaseMode) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+randomFileID+"&accessionid="+accessionID, nil) + c.Request.Header.Add("Authorization", "Bearer "+s.Token) + + accession, corrID, err := accessionMsgFileID(c, randomFileID, accessionID) + assert.Error(s.T(), err) + assert.Equal(s.T(), http.StatusNotFound, w.Code) + assert.Empty(s.T(), accession) + assert.Empty(s.T(), corrID) +} + func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" From 01afc29a4c1da20092c7c87a952226ac05a1429f Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 1 Sep 2025 16:00:23 +0200 Subject: [PATCH 016/184] Update accession endpoint tests to handle wildcard paths and improve payload handling --- sda/cmd/api/api_test.go | 144 +++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 68 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 39f2806aa..779086f80 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -475,7 +475,7 @@ func (s *TestSuite) SetupSuite() { {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, {"role":"submission","path":"/file/ingest*","action":"POST"}, - {"role":"submission","path":"/file/accession","action":"POST"}, + {"role":"submission","path":"/file/accession*","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, {"role":"submission","path":"/users/:username/file/:fileid","action":"GET"}, @@ -1289,69 +1289,80 @@ func (s *TestSuite) TestIngestMsgFileID_PayloadProvided() { assert.Empty(s.T(), corrID) } -func (s *TestSuite) TestSetAccession() { +func (s *TestSuite) TestSetAccession_WithPayload() { user := "dummy" - filePath := "/inbox/dummy/file11.c4gh" - - fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(s.T(), err, "failed to update satus of file in database") + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) - encSha := sha256.New() - _, err = encSha.Write([]byte("Checksum")) + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) assert.NoError(s.T(), err) - - decSha := sha256.New() - _, err = decSha.Write([]byte("DecryptedChecksum")) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) assert.NoError(s.T(), err) - fileInfo := database.FileInfo{ - UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), - Size: 1000, - Path: filePath, - DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), - DecryptedSize: 948, + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + "accession_id": accessionID, + }) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + r.Header.Add("Authorization", "Bearer "+s.Token) + r.Header.Set("Content-Type", "application/json") + + _, router := gin.CreateTestContext(w) + router.POST("/file/accession", rbac(e), setAccession) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) + + // verify that the message shows up in the queue + time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives + client := http.Client{Timeout: 5 * time.Second} + req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) + req.SetBasicAuth("guest", "guest") + res, err := client.Do(req) + assert.NoError(s.T(), err, "failed to query broker") + var data struct { + MessagesReady int `json:"messages_ready"` } - err = Conf.API.DB.SetArchived(fileInfo, fileID) - assert.NoError(s.T(), err, "failed to mark file as Archived") + body, err := io.ReadAll(res.Body) + res.Body.Close() + assert.NoError(s.T(), err, "failed to read response from broker") + err = json.Unmarshal(body, &data) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) +} - err = Conf.API.DB.SetVerified(fileInfo, fileID) - assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) +func (s *TestSuite) TestSetAccession_WithParams() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + fileID, _ := helperCreateVerifiedTestFile(s, user, filePath) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) - Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC model") - } + assert.NoError(s.T(), err) e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC enforcer") - } + assert.NoError(s.T(), err) - type accession struct { - AccessionID string `json:"accession_id"` - FilePath string `json:"filepath"` - User string `json:"user"` - } - aID := "API:accession-id-01" - accessionMsg, _ := json.Marshal(accession{AccessionID: aID, FilePath: filePath, User: user}) - // Mock request and response holders w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) + r := httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+fileID+"&accessionid="+accessionID, nil) r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) - router.ServeHTTP(w, r) - okResponse := w.Result() - defer okResponse.Body.Close() - assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + resp := w.Result() + defer resp.Body.Close() + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -1371,40 +1382,37 @@ func (s *TestSuite) TestSetAccession() { assert.Equal(s.T(), 1, data.MessagesReady) } -func (s *TestSuite) TestSetAccession_WrongUser() { +func (s *TestSuite) TestSetAccession_BothPayloadAndParamsProvided() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + fileID, _ := helperCreateVerifiedTestFile(s, user, filePath) + gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) - Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC model") - } + assert.NoError(s.T(), err) e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC enforcer") - } + assert.NoError(s.T(), err) + + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + "accession_id": accessionID, + }) - type accession struct { - AccessionID string `json:"accession_id"` - FilePath string `json:"filepath"` - User string `json:"user"` - } - aID := "API:accession-id-01" - accessionMsg, _ := json.Marshal(accession{AccessionID: aID, FilePath: "/inbox/dummy/file11.c4gh", User: "fooBar"}) - // Mock request and response holders w := httptest.NewRecorder() - r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) + r := httptest.NewRequest(http.MethodPost, fmt.Sprintf("/file/accession?fileid=%s&accessionid=%s", fileID, accessionID), bytes.NewBuffer(payload)) r.Header.Add("Authorization", "Bearer "+s.Token) + r.Header.Set("Content-Type", "application/json") _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) - router.ServeHTTP(w, r) - okResponse := w.Result() - defer okResponse.Body.Close() - assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + + resp := w.Result() + defer resp.Body.Close() + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) } func (s *TestSuite) TestSetAccession_WrongFormat() { From e5b292f817b2fea759339bac2a686a1a1853c615 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 2 Sep 2025 14:03:54 +0200 Subject: [PATCH 017/184] Add test for finalizing file using accession ID via file ID --- .../tests/sda/60_api_admin_test.sh | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index ecec6ff81..1b3883236 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -242,4 +242,25 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.fil done echo "Ingestion by using file ID finished successfully" +# Test giving accession id to a file by using file id +echo "Giving accession id by using file id" +# The file which ingested above will be used +accession_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/file/accession?fileid=$new_fileid&accessionid=SDA-123-asd")" +if [ "$accession_resp" != "200" ]; then + echo "Error when requesting to finalize file by the use of file id, expected 200 got: $accession_resp" + exit 1 +fi +# Check that the file has been finalized +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$new_fileid' order by started_at desc limit 1;")" = "ready" ]; do + echo "waiting for finalize to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 10 ]; then + echo "::error::Time out while waiting for finalizing to complete" + exit 1 + fi + sleep 2 +done +echo "Finalize by using file ID finished successfully" + echo "API admin tests completed successfully" From f07a437cd661924c1a947d56be7c8dd41c417e81 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 2 Sep 2025 14:17:27 +0200 Subject: [PATCH 018/184] Update file accession endpoint description and parameters for clarity --- sda/cmd/api/swagger_v1.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index dda46af7b..ac7ac782c 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -164,19 +164,40 @@ paths: description: Internal application error /file/accession: post: - description: Assigns accession ID to a given file. + description: | + Assigns accession ID to a given file. + You can provide either a JSON payload with `user`, `filepath`, and `accession_id`, + or query parameters `fileid` and `accessionid`. + If both payload and parameters are provided, a 400 Bad Request is returned. + parameters: + - in: query + name: fileid + schema: + type: string + required: false + description: UUID of the file to accession. If provided, payload must be empty. + - in: query + name: accessionid + schema: + type: string + required: false + description: Accession ID to assign to the file. If provided, payload must be empty. requestBody: content: application/json: schema: $ref: "#/components/schemas/FileAccession" + required: false responses: "200": description: Successful operation. "400": - description: Bad payload + description: | + Bad payload. Returned if both fileid/accessionid and payload are provided, or if payload is invalid. "401": description: Authentication failure. + "404": + description: File ID or decrypted checksum not found. "500": description: Internal application error. /file/ingest: From b218118af6e4599105b419976d90a0dbd617e8a1 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 2 Sep 2025 14:27:19 +0200 Subject: [PATCH 019/184] Update `/file/accession` endpoint to support query parameters and improve error handling --- sda/cmd/api/api.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 2a9d2850b..5b3a3015a 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -75,21 +75,32 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -X POST "https://HOSTNAME/file/ingest?fileid=" - `/file/accession` - - accepts `POST` requests with JSON data with the format: `{"accession_id": "", "filepath": "", "user": ""}` + - accepts `POST` requests with either: + - JSON data: `{"accession_id": "", "filepath": "", "user": ""}` + - OR query parameters: `/file/accession?fileid=&accessionid=` - assigns accession ID to the file. + - If both a JSON payload and query parameters are provided in the same request, a `400 Bad Request` is returned. + - Error codes - - `200` Query execute ok. - - `400` Error due to bad payload i.e. wrong `user` + `filepath` combination. + - `200` Query executed successfully. + - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and parameters provided, or invalid JSON). - `401` Token user is not in the list of admins. + - `404` File ID or decrypted checksum not found. - `500` Internal error due to DB or MQ failures. - Example: + Example (JSON payload): ```bash curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "my-id-01", "filepath": "/uploads/file.c4gh", "user": "testuser"}' https://HOSTNAME/file/accession ``` + Example (query parameters): + + ```bash + curl -H "Authorization: Bearer $token" -X POST "https://HOSTNAME/file/accession?fileid=&accessionid=" + ``` + - `/file/verify/:accession` - accepts `PUT` requests with an accession ID as the last element in the query - triggers re-verification of the file with the specific accession ID. From f8adf2527081c1db9e7bd7bbb465b2b36ffa5b52 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 4 Sep 2025 11:42:42 +0200 Subject: [PATCH 020/184] Close db connections in a couple of tests This needs to be done because the tests are failing due to number of connections --- sda/internal/database/db_functions_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 331957a68..4405f8bc0 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1248,6 +1248,7 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { assert.NoError(suite.T(), err) _, err = db.getInboxFilePathFromID(user, fileID) assert.Error(suite.T(), err) + db.Close() } func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { @@ -1284,6 +1285,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "archived") assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileID2) + db.Close() } func (suite *DatabaseTests) TestGetUserAndPathFromUUID_Found() { From 40f85c4c750b668260053f6e498f8345042b4008 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Mon, 8 Sep 2025 09:00:33 +0200 Subject: [PATCH 021/184] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- .github/integration/sda/rbac.json | 4 ++-- sda/cmd/api/api.go | 3 +-- sda/internal/database/database.go | 5 ----- sda/internal/database/db_functions.go | 18 +++++++++--------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 96de661b6..401f4bf8e 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -22,12 +22,12 @@ }, { "role": "submission", - "path": "/file/ingest*", + "path": "/file/ingest", "action": "POST" }, { "role": "submission", - "path": "/file/accession*", + "path": "/file/accession", "action": "POST" }, { diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f485f7188..9935fa48e 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -317,8 +317,7 @@ func getFiles(c *gin.Context) { c.JSON(200, files) } -// ingestFile function sends the ingest message -// to the broker +// ingestFile function sends the ingest message to the broker func ingestFile(c *gin.Context) { var ( ingest schema.IngestionTrigger diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index f02593e2f..bb00a755f 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -61,11 +61,6 @@ type DatasetInfo struct { Timestamp string `json:"timeStamp"` } -type IngestInfo struct { - User string - InboxPath string -} - // SchemaName is the name of the remote database schema to query var SchemaName = "sda" diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index e11dbf870..b7871c3fc 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1176,32 +1176,32 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { } // GetUserAndPathFromUUID() retrieves user and path by giving the file UUID -func (dbs *SDAdb) GetUserAndPathFromUUID(fileUUID string) (IngestInfo, error) { +func (dbs *SDAdb) GetUserAndPathFromUUID(fileUUID string) (string, string, error) { var ( - i IngestInfo + inboxPath, user string err error ) for count := 0; count <= RetryTimes; count++ { - i, err = dbs.getUserAndPathFromUUID(fileUUID) + inboxPath, user, err = dbs.getUserAndPathFromUUID(fileUUID) if err == nil { break } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } - return i, err + return inboxPath, user, err } // getUserAndPathFromUUID() is the actual function performing work for GetUserAndPathFromUUID -func (dbs *SDAdb) getUserAndPathFromUUID(fileUUID string) (IngestInfo, error) { +func (dbs *SDAdb) getUserAndPathFromUUID(fileUUID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() const query = "SELECT submission_user, submission_file_path from sda.files WHERE id = $1;" - var info IngestInfo - if err := dbs.DB.QueryRow(query, fileUUID).Scan(&info.User, &info.InboxPath); err != nil { - return IngestInfo{}, err + var inboxPath, user string + if err := dbs.DB.QueryRow(query, fileUUID).Scan(&user, &inboxPath); err != nil { + return "", "", err } - return info, nil + return inboxPath, user, nil } From bc78c32f63e774899fdb579c637bfa101cdbe5a5 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 8 Sep 2025 09:07:03 +0200 Subject: [PATCH 022/184] Remove wildcard from ingest and accession endpoint --- sda/cmd/api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 779086f80..97c7d7b6b 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -474,8 +474,8 @@ func (s *TestSuite) SetupSuite() { s.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, - {"role":"submission","path":"/file/ingest*","action":"POST"}, - {"role":"submission","path":"/file/accession*","action":"POST"}, + {"role":"submission","path":"/file/ingest","action":"POST"}, + {"role":"submission","path":"/file/accession","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, {"role":"submission","path":"/users/:username/file/:fileid","action":"GET"}, From 03596fc25280895a70e49e68109a9160f8c4bdfa Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 8 Sep 2025 13:28:49 +0200 Subject: [PATCH 023/184] Fixes for review commit --- sda/cmd/api/api.go | 12 ++++++------ sda/internal/database/db_functions_test.go | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 9935fa48e..24edbc482 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -402,7 +402,7 @@ func ingestMsgFileID(c *gin.Context, fileUUID string) (schema.IngestionTrigger, return schema.IngestionTrigger{}, "", errors.New("add either parameter or payload, not both") } // Get the user and the inbox filepath - ingestInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) + pathInfo, userInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") @@ -411,8 +411,8 @@ func ingestMsgFileID(c *gin.Context, fileUUID string) (schema.IngestionTrigger, // Information needed for the ingest message ingest.Type = "ingest" - ingest.User = ingestInfo.User - ingest.FilePath = ingestInfo.InboxPath + ingest.User = userInfo + ingest.FilePath = pathInfo // For BP the file UUID and the correlation ID are the same. // TODO: If in GDI they are not the same then change the line below corrID := fileUUID @@ -698,7 +698,7 @@ func accessionMsgFileID(c *gin.Context, fileUUID, acccessionID string) (schema.I return schema.IngestionAccession{}, "", errors.New("add either parameters or json payload, not both") } // Get the user and the inbox filepath - userPathInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) + pathInfo, userInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) if err != nil { c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") @@ -714,8 +714,8 @@ func accessionMsgFileID(c *gin.Context, fileUUID, acccessionID string) (schema.I } // Information needed for the ingest message accession.Type = "accession" - accession.User = userPathInfo.User - accession.FilePath = userPathInfo.InboxPath + accession.User = userInfo + accession.FilePath = pathInfo accession.AccessionID = acccessionID accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} // For BP the file UUID and the correlation ID are the same. diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 4405f8bc0..0edc85883 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1298,10 +1298,10 @@ func (suite *DatabaseTests) TestGetUserAndPathFromUUID_Found() { fileID, err := db.RegisterFile(filePath, user) assert.NoError(suite.T(), err, "failed to register file in database") - info, err := db.GetUserAndPathFromUUID(fileID) + pathInfo, userInfo, err := db.GetUserAndPathFromUUID(fileID) assert.NoError(suite.T(), err, "failed to get user and path from UUID") - assert.Equal(suite.T(), user, info.User) - assert.Equal(suite.T(), filePath, info.InboxPath) + assert.Equal(suite.T(), user, userInfo) + assert.Equal(suite.T(), filePath, pathInfo) db.Close() } @@ -1311,9 +1311,9 @@ func (suite *DatabaseTests) TestGetUserAndPathFromUUID_NotFound() { // Use a non-existent UUID invalidUUID := "abc-123" - info, err := db.GetUserAndPathFromUUID(invalidUUID) + pathInfo, userInfo, err := db.GetUserAndPathFromUUID(invalidUUID) assert.Error(suite.T(), err, "expected error for non-existent UUID") - assert.Empty(suite.T(), info.User) - assert.Empty(suite.T(), info.InboxPath) + assert.Empty(suite.T(), userInfo) + assert.Empty(suite.T(), pathInfo) db.Close() } From 47b5bc94dac298995f5be357d50ca835fbe7282b Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 12 Sep 2025 14:48:10 +0200 Subject: [PATCH 024/184] Refactor db function for returning structured FileDetails with correlation ID --- sda/internal/database/db_functions.go | 33 +++++++++++++++++---------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index b7871c3fc..ffc1770c7 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1175,33 +1175,42 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { return accessions, nil } -// GetUserAndPathFromUUID() retrieves user and path by giving the file UUID -func (dbs *SDAdb) GetUserAndPathFromUUID(fileUUID string) (string, string, error) { +type FileDetails struct { + User string + Path string + CorrID string +} + +// GetUserAndPathFromUUID() retrieves user, path and correlation id by giving the file UUID +func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { var ( - inboxPath, user string - err error + info FileDetails + err error ) for count := 0; count <= RetryTimes; count++ { - inboxPath, user, err = dbs.getUserAndPathFromUUID(fileUUID) + info, err = dbs.getFileDetailsFromUUID(fileUUID) if err == nil { break } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } - return inboxPath, user, err + return info, err } // getUserAndPathFromUUID() is the actual function performing work for GetUserAndPathFromUUID -func (dbs *SDAdb) getUserAndPathFromUUID(fileUUID string) (string, string, error) { +func (dbs *SDAdb) getFileDetailsFromUUID(fileUUID string) (FileDetails, error) { + var info FileDetails dbs.checkAndReconnectIfNeeded() - const query = "SELECT submission_user, submission_file_path from sda.files WHERE id = $1;" - var inboxPath, user string - if err := dbs.DB.QueryRow(query, fileUUID).Scan(&user, &inboxPath); err != nil { - return "", "", err + const query = `SELECT f.submission_user, f.submission_file_path, fel.correlation_id + from sda.files f + join sda.file_event_log fel on f.id = fel.file_id + WHERE f.id = $1 and fel.event='uploaded';` + if err := dbs.DB.QueryRow(query, fileUUID).Scan(&info.User, &info.Path, &info.CorrID); err != nil { + return FileDetails{}, err } - return inboxPath, user, nil + return info, nil } From f604d5dc2faa597b65d461618297cba89750017e Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 12 Sep 2025 14:48:34 +0200 Subject: [PATCH 025/184] Update db function test --- sda/internal/database/db_functions_test.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 0edc85883..cc177bd07 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1288,7 +1288,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { db.Close() } -func (suite *DatabaseTests) TestGetUserAndPathFromUUID_Found() { +func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") @@ -1298,10 +1298,16 @@ func (suite *DatabaseTests) TestGetUserAndPathFromUUID_Found() { fileID, err := db.RegisterFile(filePath, user) assert.NoError(suite.T(), err, "failed to register file in database") - pathInfo, userInfo, err := db.GetUserAndPathFromUUID(fileID) + // Update event log to ensure correlation ID is set + correlationID := "b7e2c1a4-5f3b-4c8e-9d2a-7f6e1b2c3d4e" + err = db.UpdateFileEventLog(fileID, "uploaded", correlationID, user, "{}", "{}") + assert.NoError(suite.T(), err, "failed to update file event log") + + infoFile, err := db.GetFileDetailsFromUUID(fileID) assert.NoError(suite.T(), err, "failed to get user and path from UUID") - assert.Equal(suite.T(), user, userInfo) - assert.Equal(suite.T(), filePath, pathInfo) + assert.Equal(suite.T(), user, infoFile.User) + assert.Equal(suite.T(), filePath, infoFile.Path) + assert.Equal(suite.T(), correlationID, infoFile.CorrID) db.Close() } @@ -1311,9 +1317,10 @@ func (suite *DatabaseTests) TestGetUserAndPathFromUUID_NotFound() { // Use a non-existent UUID invalidUUID := "abc-123" - pathInfo, userInfo, err := db.GetUserAndPathFromUUID(invalidUUID) + infoFile, err := db.GetFileDetailsFromUUID(invalidUUID) assert.Error(suite.T(), err, "expected error for non-existent UUID") - assert.Empty(suite.T(), userInfo) - assert.Empty(suite.T(), pathInfo) + assert.Empty(suite.T(), infoFile.User) + assert.Empty(suite.T(), infoFile.Path) + assert.Empty(suite.T(), infoFile.CorrID) db.Close() } From 8482f686bc5d6478b5acef267762f917d13e78e8 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 12 Sep 2025 14:51:36 +0200 Subject: [PATCH 026/184] Refactor ingestFile and setAccession functions Remove the external functions and handle bothe cases in the same function (use payload or parameters) --- sda/cmd/api/api.go | 286 ++++++++++++++++++--------------------------- 1 file changed, 114 insertions(+), 172 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 24edbc482..164172d36 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -317,25 +317,65 @@ func getFiles(c *gin.Context) { c.JSON(200, files) } -// ingestFile function sends the ingest message to the broker +/* +ingestFile handles requests to initiate ingestion of a file. +This endpoint supports two input modes: +1. By file ID (via the "fileid" query parameter): Looks up the user and file path from the database. +2. By JSON payload: Expects a JSON body with user and file path. +The function constructs an ingest message, validates it +and sends it to the broker with the appropriate correlation ID. +*/ func ingestFile(c *gin.Context) { var ( ingest schema.IngestionTrigger corrID string - err error ) - fileID := c.Query("fileid") - if fileID == "" { - ingest, corrID, err = ingestMsgFilePath(c) + hasQuery := c.Query("fileid") != "" + hasBody := c.Request.ContentLength > 0 + switch { + case hasQuery && hasBody: + c.AbortWithStatusJSON(http.StatusBadRequest, "both file ID parameter and payload provided. Choose one") + + return + case hasQuery: + // Get the user and the inbox filepath + fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "file information not found") + return } - } else { - ingest, corrID, err = ingestMsgFileID(c, fileID) + // Add file info in the message payload + ingest.User = fileDetails.User + ingest.FilePath = fileDetails.Path + corrID = fileDetails.CorrID + default: + // Bind ingest and payload + if err = c.BindJSON(&ingest); err != nil { + c.AbortWithStatusJSON( + http.StatusBadRequest, + gin.H{ + "error": "json decoding : " + err.Error(), + "status": http.StatusBadRequest, + }, + ) + + return + } + // Find the correlation id of the file + corrID, err = Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") if err != nil { + if corrID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + } else { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + } + return } } + // Add type in message payload + ingest.Type = "ingest" marshaledMsg, _ := json.Marshal(&ingest) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { @@ -354,72 +394,6 @@ func ingestFile(c *gin.Context) { c.Status(http.StatusOK) } -// ingestMsgFilePath parses the JSON payload from the request to construct an ingestion trigger message. -// It validates the payload and retrieves the correlation ID for the file using the user and file path. -// Returns the parsed ingestion trigger struct, the correlation ID, and an error if any occurred. -func ingestMsgFilePath(c *gin.Context) (schema.IngestionTrigger, string, error) { - var ingest schema.IngestionTrigger - // Bind ingest and payload - if err := c.BindJSON(&ingest); err != nil { - c.AbortWithStatusJSON( - http.StatusBadRequest, - gin.H{ - "error": "json decoding : " + err.Error(), - "status": http.StatusBadRequest, - }, - ) - - return schema.IngestionTrigger{}, "", err - } - // Find the correlation id of the file - corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") - if err != nil { - if corrID == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - } else { - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - } - - return schema.IngestionTrigger{}, "", err - } - // Add type in message payload - ingest.Type = "ingest" - - return ingest, corrID, nil -} - -// ingestMsgFileID constructs an ingestion trigger message using a file UUID provided as a query parameter. -// It checks that no JSON payload is present in the request; if both are provided, it returns a 400 Bad Request. -// The function retrieves the user and file path from the database using the file UUID. -// If the file UUID is not found, it responds with a 404 Not Found error. -// Returns the constructed ingestion trigger struct, the correlation ID (file UUID), and an error if any occurred. -func ingestMsgFileID(c *gin.Context, fileUUID string) (schema.IngestionTrigger, string, error) { - var ingest schema.IngestionTrigger - // Check if payload is provided as well - if c.Request.ContentLength > 0 { - c.AbortWithStatusJSON(http.StatusBadRequest, "Both file ID parameter and payload provided. Choose one") - - return schema.IngestionTrigger{}, "", errors.New("add either parameter or payload, not both") - } - // Get the user and the inbox filepath - pathInfo, userInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) - if err != nil { - c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") - - return schema.IngestionTrigger{}, "", err - } - - // Information needed for the ingest message - ingest.Type = "ingest" - ingest.User = userInfo - ingest.FilePath = pathInfo - // For BP the file UUID and the correlation ID are the same. - // TODO: If in GDI they are not the same then change the line below - corrID := fileUUID - - return ingest, corrID, nil -} - // The deleteFile function deletes files from the inbox and marks them as // discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { @@ -594,30 +568,85 @@ func downloadFile(c *gin.Context) { c.Status(http.StatusOK) } -// setAccession handles requests to assign an accession ID to a file. -// It accepts either a JSON payload with user and file path, or query parameters for fileid and accessionid. -// If both payload and parameters are provided, a 400 Bad Request is returned. -// The function validates the request, constructs the accession message, and sends it to the broker. -// Returns 200 OK on success, or an appropriate error code on failure. +/* +setAccession handles requests to assign an accession ID to a file. +This endpoint supports two input modes: +1. By query parameters ("fileid" and "accessionid"): Retrieves user, file path, and decrypted checksum from the database using the file ID. +2. By JSON payload: Expects a JSON body with user and file path, then looks up the correlation ID and decrypted checksum. +If both query parameters and a JSON payload are provided, the request is rejected with a 400 Bad Request. +The function constructs an accession message, validates it and sends it to the message broker. +*/ func setAccession(c *gin.Context) { var ( accession schema.IngestionAccession corrID string - err error ) - accessionID := c.Query("accessionid") - fileID := c.Query("fileid") - if fileID == "" || accessionID == "" { - accession, corrID, err = accessionMsgFilePath(c) + hasQuery := c.Query("fileid") != "" || c.Query("accessionid") != "" + hasBody := c.Request.ContentLength > 0 + switch { + case hasQuery && hasBody: + c.AbortWithStatusJSON(http.StatusBadRequest, "both parameters and json payload provided. Choose one") + + return + case hasQuery: + // Get the user and the inbox filepath + fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) if err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "file details not found") + return } - } else { - accession, corrID, err = accessionMsgFileID(c, fileID, accessionID) + // Get the decrypted checksum + fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(c.Query("fileid")) if err != nil { + log.Debugln(err.Error()) + c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") + return } + // Add info in message payload + accession.AccessionID = c.Query("accessionid") + accession.User = fileDetails.User + accession.FilePath = fileDetails.Path + accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} + // Corellation id + corrID = fileDetails.CorrID + default: + if err = c.BindJSON(&accession); err != nil { + c.AbortWithStatusJSON( + http.StatusBadRequest, + gin.H{ + "error": "json decoding : " + err.Error(), + "status": http.StatusBadRequest, + }, + ) + + return + } + // Find the correlation id + corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") + if err != nil { + if corrID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + } else { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + } + + return + } + // Get decrypted checksum + fileInfo, err := Conf.API.DB.GetFileInfo(corrID) + if err != nil { + log.Debugln(err.Error()) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + // Add decrypted checksum in message payload + accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}} } + // Add type in the message payload + accession.Type = "accession" marshaledMsg, _ := json.Marshal(&accession) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { @@ -638,93 +667,6 @@ func setAccession(c *gin.Context) { c.Status(http.StatusOK) } -// accessionMsgFilePath parses the JSON payload from the request. -// It validates the payload and retrieves all the information needed -// to construct the accession payload message. -// Returns the accession payload, the correlation ID, and an error if any occurred. -func accessionMsgFilePath(c *gin.Context) (schema.IngestionAccession, string, error) { - var accession schema.IngestionAccession - if err := c.BindJSON(&accession); err != nil { - c.AbortWithStatusJSON( - http.StatusBadRequest, - gin.H{ - "error": "json decoding : " + err.Error(), - "status": http.StatusBadRequest, - }, - ) - - return schema.IngestionAccession{}, "", err - } - - corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") - if err != nil { - if corrID == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - } else { - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - } - - return schema.IngestionAccession{}, "", err - } - - // For the BP case the correlation id is the same with the file id. - // TODO: If in GDi the IDs are different need to find the file id - fileUUID := corrID - fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileUUID) - if err != nil { - log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") - - return schema.IngestionAccession{}, "", err - } - - accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} - accession.Type = "accession" - - return accession, corrID, nil -} - -// accessionMsgFileID constructs an accession message using a file UUID and accession ID provided as parameters. -// It checks that no JSON payload is present in the request; if both are provided, it returns a 400 Bad Request. -// The function retrieves the user, file path, and decrypted checksum from the database using the file UUID. -// If the file UUID or checksum is not found, it responds with a 404 Not Found error. -// Returns the accession payload, the correlation ID, and an error if any occurred. -func accessionMsgFileID(c *gin.Context, fileUUID, acccessionID string) (schema.IngestionAccession, string, error) { - var accession schema.IngestionAccession - // Check if payload is provided as well - if c.Request.ContentLength > 0 { - c.AbortWithStatusJSON(http.StatusBadRequest, "Both parameters and json payload provided. Choose one") - - return schema.IngestionAccession{}, "", errors.New("add either parameters or json payload, not both") - } - // Get the user and the inbox filepath - pathInfo, userInfo, err := Conf.API.DB.GetUserAndPathFromUUID(fileUUID) - if err != nil { - c.AbortWithStatusJSON(http.StatusNotFound, "file ID not found") - - return schema.IngestionAccession{}, "", err - } - // Get the decrypted checksum - fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileUUID) - if err != nil { - log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") - - return schema.IngestionAccession{}, "", err - } - // Information needed for the ingest message - accession.Type = "accession" - accession.User = userInfo - accession.FilePath = pathInfo - accession.AccessionID = acccessionID - accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} - // For BP the file UUID and the correlation ID are the same. - // TODO: If in GDI they are not the same then change the line below - corrID := fileUUID - - return accession, corrID, nil -} - func createDataset(c *gin.Context) { var dataset dataset if err := c.BindJSON(&dataset); err != nil { From fb3b7165c96affa2bed79d24f3b470ef51a68ea0 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 12 Sep 2025 14:52:27 +0200 Subject: [PATCH 027/184] Update unit tests --- sda/cmd/api/api_test.go | 427 ++++++++++++++++++++++------------------ 1 file changed, 235 insertions(+), 192 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 97c7d7b6b..d13b6712c 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1068,6 +1068,7 @@ func (s *TestSuite) TestRBAC_emptyPolicy() { assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) assert.Contains(s.T(), string(b), "not authorized") } + func (s *TestSuite) TestIngestFile_WithPayload() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" @@ -1127,6 +1128,118 @@ func (s *TestSuite) TestIngestFile_WithPayload() { assert.Equal(s.T(), 1, data.MessagesReady) } +func (s *TestSuite) TestIngestFile_WithPayload_NoUser() { + user := "dummy" + filePath := "/inbox/dummy/file10.c4gh" + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + Conf.Broker.SchemasPath = "../../schemas/isolated" + + type ingest struct { + FilePath string `json:"filepath"` + User string `json:"user"` + } + ingestMsg, _ := json.Marshal(ingest{User: "", FilePath: filePath}) + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.POST("/file/ingest", rbac(e), ingestFile) + + router.ServeHTTP(w, r) + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") +} + +func (s *TestSuite) TestIngestFile_WithPayload_WrongUser() { + user := "dummy" + filePath := "/inbox/dummy/file10.c4gh" + + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + Conf.Broker.SchemasPath = "../../schemas/isolated" + + type ingest struct { + FilePath string `json:"filepath"` + User string `json:"user"` + } + ingestMsg, _ := json.Marshal(ingest{User: "foo", FilePath: filePath}) + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) + + _, router := gin.CreateTestContext(w) + router.POST("/file/ingest", ingestFile) + + router.ServeHTTP(w, r) + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") +} + +func (s *TestSuite) TestIngestFile_WrongFilePath() { + user := "dummy" + filePath := "/inbox/dummy/file10.c4gh" + + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + + Conf.Broker.SchemasPath = "../../schemas/isolated" + + type ingest struct { + FilePath string `json:"filepath"` + User string `json:"user"` + } + ingestMsg, _ := json.Marshal(ingest{User: "dummy", FilePath: "bad/path"}) + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest("POST", "/file/ingest", bytes.NewBuffer(ingestMsg)) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.POST("/file/ingest", ingestFile) + + router.ServeHTTP(w, r) + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") +} + func (s *TestSuite) TestIngestFile_WithFileID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" @@ -1172,6 +1285,32 @@ func (s *TestSuite) TestIngestFile_WithFileID() { assert.Equal(s.T(), 1, data.MessagesReady) } +func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { + user := "dummy" + filePath := "/inbox/dummy/file11.c4gh" + fileID, err := Conf.API.DB.RegisterFile(filePath, user) + assert.NoError(s.T(), err) + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + assert.NoError(s.T(), err) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + + w := httptest.NewRecorder() + r := httptest.NewRequest("POST", "/file/ingest?fileid=random-1234", nil) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.POST("/file/ingest", ingestFile) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "file information not found") +} + func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" @@ -1204,7 +1343,7 @@ func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { defer resp.Body.Close() body, _ := io.ReadAll(resp.Body) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) - assert.Contains(s.T(), string(body), "Both file ID parameter and payload provided") + assert.Contains(s.T(), string(body), "both file ID parameter and payload provided") } func (s *TestSuite) TestIngestFile_NoFileIDnoPayload() { @@ -1221,75 +1360,92 @@ func (s *TestSuite) TestIngestFile_NoFileIDnoPayload() { assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (s *TestSuite) TestIngestMsgFileID() { +func (s *TestSuite) TestSetAccession_WithPayload() { user := "dummy" filePath := "/inbox/dummy_folder/dummyfile.c4gh" - - fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest("POST", "/file/ingest", nil) - - ingest, corrID, err := ingestMsgFileID(c, fileID) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) assert.NoError(s.T(), err) - assert.Equal(s.T(), user, ingest.User) - assert.Equal(s.T(), filePath, ingest.FilePath) - assert.Equal(s.T(), fileID, corrID) -} - -func (s *TestSuite) TestIngestMsgFileID_NotFound() { - user := "dummy" - filePath := "/inbox/dummy/file10.c4gh" - _, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") + payload, _ := json.Marshal(map[string]string{ + "user": user, + "filepath": filePath, + "accession_id": accessionID, + }) - gin.SetMode(gin.ReleaseMode) - assert.NoError(s.T(), setupJwtAuth()) w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest("POST", "/file/ingest", nil) + r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + r.Header.Add("Authorization", "Bearer "+s.Token) + r.Header.Set("Content-Type", "application/json") + + _, router := gin.CreateTestContext(w) + router.POST("/file/accession", rbac(e), setAccession) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) - ingest, corrID, err := ingestMsgFileID(c, "random-id") - assert.Error(s.T(), err) - assert.Contains(s.T(), w.Body.String(), "file ID not found") - assert.Equal(s.T(), http.StatusNotFound, w.Code) - assert.Empty(s.T(), ingest) - assert.Empty(s.T(), corrID) + // verify that the message shows up in the queue + time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives + client := http.Client{Timeout: 5 * time.Second} + req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) + req.SetBasicAuth("guest", "guest") + res, err := client.Do(req) + assert.NoError(s.T(), err, "failed to query broker") + var data struct { + MessagesReady int `json:"messages_ready"` + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + assert.NoError(s.T(), err, "failed to read response from broker") + err = json.Unmarshal(body, &data) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (s *TestSuite) TestIngestMsgFileID_PayloadProvided() { +func (s *TestSuite) TestSetAccession_WithPayload_WrongUser() { user := "dummy" filePath := "/inbox/dummy_folder/dummyfile.c4gh" - - fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(s.T(), err, "failed to register file in database") + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + assert.NoError(s.T(), err) payload, _ := json.Marshal(map[string]string{ - "user": user, - "filepath": filePath, + "user": "Foo-bar", + "filepath": filePath, + "accession_id": accessionID, }) - c.Request = httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(payload)) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - ingest, corrID, err := ingestMsgFileID(c, fileID) - assert.Error(s.T(), err) - assert.Contains(s.T(), w.Body.String(), "Both file ID parameter and payload provided. Choose one") - assert.Equal(s.T(), http.StatusBadRequest, w.Code) - assert.Empty(s.T(), ingest) - assert.Empty(s.T(), corrID) + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) + r.Header.Add("Authorization", "Bearer "+s.Token) + r.Header.Set("Content-Type", "application/json") + + _, router := gin.CreateTestContext(w) + router.POST("/file/accession", rbac(e), setAccession) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } -func (s *TestSuite) TestSetAccession_WithPayload() { +func (s *TestSuite) TestSetAccession_WithPayload_WrongPath() { user := "dummy" filePath := "/inbox/dummy_folder/dummyfile.c4gh" accessionID := "accession-id-01" @@ -1304,7 +1460,7 @@ func (s *TestSuite) TestSetAccession_WithPayload() { payload, _ := json.Marshal(map[string]string{ "user": user, - "filepath": filePath, + "filepath": "/inbox/random/path/foo.c4gh", "accession_id": accessionID, }) @@ -1319,24 +1475,9 @@ func (s *TestSuite) TestSetAccession_WithPayload() { resp := w.Result() defer resp.Body.Close() - assert.Equal(s.T(), http.StatusOK, resp.StatusCode) - - // verify that the message shows up in the queue - time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives - client := http.Client{Timeout: 5 * time.Second} - req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) - req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) - assert.NoError(s.T(), err, "failed to query broker") - var data struct { - MessagesReady int `json:"messages_ready"` - } - body, err := io.ReadAll(res.Body) - res.Body.Close() - assert.NoError(s.T(), err, "failed to read response from broker") - err = json.Unmarshal(body, &data) - assert.NoError(s.T(), err, "failed to unmarshal response") - assert.Equal(s.T(), 1, data.MessagesReady) + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } func (s *TestSuite) TestSetAccession_WithParams() { @@ -1382,6 +1523,34 @@ func (s *TestSuite) TestSetAccession_WithParams() { assert.Equal(s.T(), 1, data.MessagesReady) } +func (s *TestSuite) TestSetAccession_WithParams_WrongID() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + accessionID := "accession-id-01" + _, _ = helperCreateVerifiedTestFile(s, user, filePath) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + assert.NoError(s.T(), err) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/accession?fileid=randomID-1234&accessionid="+accessionID, nil) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.POST("/file/accession", rbac(e), setAccession) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "file details not found") +} + func (s *TestSuite) TestSetAccession_BothPayloadAndParamsProvided() { user := "dummy" filePath := "/inbox/dummy_folder/dummyfile.c4gh" @@ -1451,132 +1620,6 @@ func (s *TestSuite) TestSetAccession_WrongFormat() { assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (s *TestSuite) TestAccessionMsgFilePath() { - user := "dummy" - filePath := "/inbox/dummy_folder/dummyfile.c4gh" - accessionID := "accession-id-01" - fileID, decSha := helperCreateVerifiedTestFile(s, user, filePath) - - gin.SetMode(gin.ReleaseMode) - assert.NoError(s.T(), setupJwtAuth()) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - - payload, _ := json.Marshal(map[string]string{ - "user": user, - "filepath": filePath, - "accession_id": accessionID, - }) - c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - - accession, corrID, err := accessionMsgFilePath(c) - assert.NoError(s.T(), err) - assert.Equal(s.T(), user, accession.User) - assert.Equal(s.T(), filePath, accession.FilePath) - assert.Equal(s.T(), accessionID, accession.AccessionID) - assert.Equal(s.T(), fmt.Sprintf("%x", decSha.Sum(nil)), accession.DecryptedChecksums[0].Value) - assert.Equal(s.T(), fileID, corrID) -} - -func (s *TestSuite) TestAccessionMsgFilePath_WrongUser() { - user := "dummy" - filePath := "/inbox/dummy_folder/dummyfile.c4gh" - accessionID := "accession-id-01" - _, _ = helperCreateVerifiedTestFile(s, user, filePath) - - gin.SetMode(gin.ReleaseMode) - assert.NoError(s.T(), setupJwtAuth()) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - - payload, _ := json.Marshal(map[string]string{ - "user": "no-dummy-user", - "filepath": filePath, - "accession_id": accessionID, - }) - c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - - accession, corrID, err := accessionMsgFilePath(c) - assert.Error(s.T(), err) - assert.Equal(s.T(), http.StatusBadRequest, w.Code) - assert.Empty(s.T(), accession) - assert.Empty(s.T(), corrID) -} - -func (s *TestSuite) TestAccessionMsgFilePath_WrongPath() { - user := "dummy" - filePath := "/inbox/dummy_folder/dummyfile.c4gh" - accessionID := "accession-id-01" - _, _ = helperCreateVerifiedTestFile(s, user, filePath) - - gin.SetMode(gin.ReleaseMode) - assert.NoError(s.T(), setupJwtAuth()) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - - payload, _ := json.Marshal(map[string]string{ - "user": user, - "filepath": "random/folder/dumfile.c4gh", - "accession_id": accessionID, - }) - c.Request = httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(payload)) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - - accession, corrID, err := accessionMsgFilePath(c) - assert.Error(s.T(), err) - assert.Equal(s.T(), http.StatusBadRequest, w.Code) - assert.Empty(s.T(), accession) - assert.Empty(s.T(), corrID) -} - -func (s *TestSuite) TestAccessionMsgFileID() { - user := "dummy" - filePath := "/inbox/dummy_folder/dummyfile.c4gh" - accessionID := "accession-id-01" - - // Create and verify test file - fileID, decSha := helperCreateVerifiedTestFile(s, user, filePath) - - // Set up Gin context with no payload - gin.SetMode(gin.ReleaseMode) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+fileID+"&accessionid="+accessionID, nil) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - - accession, corrID, err := accessionMsgFileID(c, fileID, accessionID) - assert.NoError(s.T(), err) - assert.Equal(s.T(), user, accession.User) - assert.Equal(s.T(), filePath, accession.FilePath) - assert.Equal(s.T(), accessionID, accession.AccessionID) - assert.Equal(s.T(), "accession", accession.Type) - assert.Equal(s.T(), fileID, corrID) - assert.Equal(s.T(), fmt.Sprintf("%x", decSha.Sum(nil)), accession.DecryptedChecksums[0].Value) -} - -func (s *TestSuite) TestAccessionMsgFileID_WrongFileID() { - user := "dummy" - filePath := "/inbox/dummy_folder/dummyfile.c4gh" - accessionID := "accession-id-01" - // Register file and then use a random fileID - _, _ = helperCreateVerifiedTestFile(s, user, filePath) - randomFileID := "non-existent-file-id" - - gin.SetMode(gin.ReleaseMode) - w := httptest.NewRecorder() - c, _ := gin.CreateTestContext(w) - c.Request = httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+randomFileID+"&accessionid="+accessionID, nil) - c.Request.Header.Add("Authorization", "Bearer "+s.Token) - - accession, corrID, err := accessionMsgFileID(c, randomFileID, accessionID) - assert.Error(s.T(), err) - assert.Equal(s.T(), http.StatusNotFound, w.Code) - assert.Empty(s.T(), accession) - assert.Empty(s.T(), corrID) -} - func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" From b6fac68253e8a078d38cda24409c37a2b471daa7 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 12 Sep 2025 15:32:54 +0200 Subject: [PATCH 028/184] Update document and swagger --- sda/cmd/api/api.md | 7 +++---- sda/cmd/api/swagger_v1.yml | 10 ++++------ 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 5b3a3015a..e628dce80 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -58,9 +58,8 @@ Admin endpoints are only available to a set of whitelisted users specified in th - Error codes - `200` Query executed successfully. - - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and fileid provided, or invalid JSON). + - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and fileid provided, invalid fileid, or invalid JSON). - `401` Token user is not in the list of admins. - - `404` File ID not found. - `500` Internal error due to DB or MQ failures. Example (JSON payload): @@ -84,9 +83,9 @@ Admin endpoints are only available to a set of whitelisted users specified in th - Error codes - `200` Query executed successfully. - - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and parameters provided, or invalid JSON). + - `400` Bad request (e.g. wrong `user` + `filepath` combination, both payload and parameters provided, invalid fileid, or invalid JSON). - `401` Token user is not in the list of admins. - - `404` File ID or decrypted checksum not found. + - `404` Decrypted checksum not found. - `500` Internal error due to DB or MQ failures. Example (JSON payload): diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index ac7ac782c..bea8b78f5 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -193,11 +193,11 @@ paths: description: Successful operation. "400": description: | - Bad payload. Returned if both fileid/accessionid and payload are provided, or if payload is invalid. + Bad request. Returned if both fileid/accessionid and payload are provided, or if payload is invalid, or if fileid is invalid. "401": description: Authentication failure. "404": - description: File ID or decrypted checksum not found. + description: Decrypted checksum not found. "500": description: Internal application error. /file/ingest: @@ -224,11 +224,9 @@ paths: description: Successful operation. "400": description: | - Bad payload. Returned if both fileid and payload are provided, or if payload is invalid. + Bad request. Returned if both fileid and payload are provided, or if payload is invalid, or if fileid is invalid. "401": description: Authentication failure. - "404": - description: File ID not found. "500": description: Internal application error. /file/{userName}/{fileID}: @@ -480,4 +478,4 @@ components: scheme: bearer bearerFormat: JWT security: - - bearerAuth: [] \ No newline at end of file + - bearerAuth: [] From 21fdb08a3f485d5862994683a9961e910228e411 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 15 Sep 2025 00:11:19 +0200 Subject: [PATCH 029/184] Update integration test --- .github/integration/tests/sda/60_api_admin_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 1b3883236..a81b7be93 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -222,9 +222,9 @@ echo "Ingest file by using file ID" s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ingest/NB12878-ingest.bam.c4gh sleep 3 # Find the file id of the uploaded file -new_fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/ingest/NB12878-ingest.bam.c4gh") | .fileID')" +new_fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "ingest/NB12878-ingest.bam.c4gh") | .fileID')" # ingest the file -ingest_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/file/ingest?fileid=$new_fileid")" +ingest_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X POST "http://api:8080/file/ingest?fileid=$new_fileid")" if [ "$ingest_resp" != "200" ]; then echo "Error when requesting to ingesting file by the use of file id, expected 200 got: $ingest_resp" exit 1 From 73261f3cc9b0006f2ccba00e49fa386bd259f482 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 15 Sep 2025 11:24:39 +0200 Subject: [PATCH 030/184] Add validation for missing accession ID in setAccession function and corresponding test --- sda/cmd/api/api.go | 5 +++++ sda/cmd/api/api_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 164172d36..1d939b723 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -582,11 +582,16 @@ func setAccession(c *gin.Context) { corrID string ) hasQuery := c.Query("fileid") != "" || c.Query("accessionid") != "" + missingAccession := c.Query("fileid") != "" && c.Query("accessionid") == "" hasBody := c.Request.ContentLength > 0 switch { case hasQuery && hasBody: c.AbortWithStatusJSON(http.StatusBadRequest, "both parameters and json payload provided. Choose one") + return + case missingAccession: + c.AbortWithStatusJSON(http.StatusBadRequest, "accessionid is not provided") + return case hasQuery: // Get the user and the inbox filepath diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index d13b6712c..49d4f967e 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1551,6 +1551,33 @@ func (s *TestSuite) TestSetAccession_WithParams_WrongID() { assert.Contains(s.T(), string(b), "file details not found") } +func (s *TestSuite) TestSetAccession_WithParams_MissingAccession() { + user := "dummy" + filePath := "/inbox/dummy_folder/dummyfile.c4gh" + fileID, _ := helperCreateVerifiedTestFile(s, user, filePath) + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + assert.NoError(s.T(), err) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + assert.NoError(s.T(), err) + + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodPost, "/file/accession?fileid="+fileID, nil) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.POST("/file/accession", rbac(e), setAccession) + router.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + b, _ := io.ReadAll(resp.Body) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(b), "accessionid is not provided") +} + func (s *TestSuite) TestSetAccession_BothPayloadAndParamsProvided() { user := "dummy" filePath := "/inbox/dummy_folder/dummyfile.c4gh" From d099de4517a52567cbf47aba80ad0630cd133344 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 15 Sep 2025 14:36:50 +0200 Subject: [PATCH 031/184] Move FileDetails struct definition to database.go --- sda/internal/database/database.go | 6 ++++++ sda/internal/database/db_functions.go | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index bb00a755f..82e7da664 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -61,6 +61,12 @@ type DatasetInfo struct { Timestamp string `json:"timeStamp"` } +type FileDetails struct { + User string + Path string + CorrID string +} + // SchemaName is the name of the remote database schema to query var SchemaName = "sda" diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index ffc1770c7..da653382a 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1175,12 +1175,6 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { return accessions, nil } -type FileDetails struct { - User string - Path string - CorrID string -} - // GetUserAndPathFromUUID() retrieves user, path and correlation id by giving the file UUID func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { var ( From f85d1a6d7841e656a94cf52a1f5604823440ef6a Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Mon, 15 Sep 2025 14:48:14 +0200 Subject: [PATCH 032/184] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 1d939b723..5479d5934 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -330,14 +330,12 @@ func ingestFile(c *gin.Context) { ingest schema.IngestionTrigger corrID string ) - hasQuery := c.Query("fileid") != "" - hasBody := c.Request.ContentLength > 0 switch { - case hasQuery && hasBody: - c.AbortWithStatusJSON(http.StatusBadRequest, "both file ID parameter and payload provided. Choose one") + case c.Query("fileid") != "" && c.Request.ContentLength > 0: + c.AbortWithStatusJSON(http.StatusBadRequest, "both file ID parameter and payload provided.") return - case hasQuery: + case c.Query("fileid") != "": // Get the user and the inbox filepath fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) if err != nil { @@ -349,7 +347,7 @@ func ingestFile(c *gin.Context) { ingest.User = fileDetails.User ingest.FilePath = fileDetails.Path corrID = fileDetails.CorrID - default: + case c.Request.ContentLength > 0: // Bind ingest and payload if err = c.BindJSON(&ingest); err != nil { c.AbortWithStatusJSON( @@ -373,6 +371,8 @@ func ingestFile(c *gin.Context) { return } + default: + c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") } // Add type in message payload ingest.Type = "ingest" @@ -593,7 +593,7 @@ func setAccession(c *gin.Context) { c.AbortWithStatusJSON(http.StatusBadRequest, "accessionid is not provided") return - case hasQuery: + case c.Query("fileid") != "" && c.Query("accessionid") != "": // Get the user and the inbox filepath fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) if err != nil { @@ -616,7 +616,7 @@ func setAccession(c *gin.Context) { accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} // Corellation id corrID = fileDetails.CorrID - default: + case c.Request.ContentLength > 0: if err = c.BindJSON(&accession); err != nil { c.AbortWithStatusJSON( http.StatusBadRequest, From 73c8b40e62f0661fbd05557c312bf826c0cf5da4 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 15 Sep 2025 15:23:33 +0200 Subject: [PATCH 033/184] Small fixes --- sda/cmd/api/api.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 5479d5934..4b9fc7e8a 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -371,8 +371,8 @@ func ingestFile(c *gin.Context) { return } - default: - c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") + default: + c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") } // Add type in message payload ingest.Type = "ingest" @@ -649,6 +649,10 @@ func setAccession(c *gin.Context) { } // Add decrypted checksum in message payload accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}} + default: + c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") + + return } // Add type in the message payload accession.Type = "accession" From 64517ef285845890239a325a76316d1dee24e998 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Mon, 15 Sep 2025 15:59:35 +0200 Subject: [PATCH 034/184] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 4b9fc7e8a..a2bd81de9 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -629,9 +629,9 @@ func setAccession(c *gin.Context) { return } // Find the correlation id - corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") + fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "uploaded") if err != nil { - if corrID == "" { + if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -640,7 +640,7 @@ func setAccession(c *gin.Context) { return } // Get decrypted checksum - fileInfo, err := Conf.API.DB.GetFileInfo(corrID) + fileInfo, err := Conf.API.DB.GetFileInfo(fileID) if err != nil { log.Debugln(err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -657,6 +657,17 @@ func setAccession(c *gin.Context) { // Add type in the message payload accession.Type = "accession" + corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") + if err != nil { + if corrID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + } else { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + } + + return + } + marshaledMsg, _ := json.Marshal(&accession) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { log.Debugln(err.Error()) From a8b0a227776b2d76a1f7f126f13f3e69f30d20c5 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Mon, 15 Sep 2025 16:06:18 +0200 Subject: [PATCH 035/184] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/internal/database/db_functions_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index cc177bd07..dcc17ce1e 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1296,12 +1296,16 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { filePath := "/dummy_user.org/Dummy_folder/dummyfile.c4gh" user := "dummy@user.org" fileID, err := db.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + if err != nil { + suite.FailNow("failed to register file in database") + } // Update event log to ensure correlation ID is set correlationID := "b7e2c1a4-5f3b-4c8e-9d2a-7f6e1b2c3d4e" err = db.UpdateFileEventLog(fileID, "uploaded", correlationID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update file event log") + if err != nil { + suite.FailNow("failed to update file event log") + } infoFile, err := db.GetFileDetailsFromUUID(fileID) assert.NoError(suite.T(), err, "failed to get user and path from UUID") From 1310410cd28530036c1d4511df934fbad774b585 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 15 Sep 2025 16:43:59 +0200 Subject: [PATCH 036/184] Fixes mainly for correlation id and checksum --- sda/cmd/api/api.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index a2bd81de9..80e4b2269 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -629,7 +629,7 @@ func setAccession(c *gin.Context) { return } // Find the correlation id - fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "uploaded") + fileID, err := Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "uploaded") if err != nil { if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) @@ -639,16 +639,27 @@ func setAccession(c *gin.Context) { return } + // Get correlation id + corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") + if err != nil { + if corrID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) + } else { + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + } + + return + } // Get decrypted checksum - fileInfo, err := Conf.API.DB.GetFileInfo(fileID) + fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileID) if err != nil { log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") return } // Add decrypted checksum in message payload - accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileInfo.DecryptedChecksum}} + accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} default: c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") @@ -657,17 +668,6 @@ func setAccession(c *gin.Context) { // Add type in the message payload accession.Type = "accession" - corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") - if err != nil { - if corrID == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - } else { - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - } - - return - } - marshaledMsg, _ := json.Marshal(&accession) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { log.Debugln(err.Error()) From a39cf54f5ee235c75ee91e224963b2e7c4ba2fa7 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Wed, 17 Sep 2025 10:43:24 +0200 Subject: [PATCH 037/184] Add return in the default case of ingestFile --- sda/cmd/api/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 80e4b2269..1d16b992e 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -373,6 +373,8 @@ func ingestFile(c *gin.Context) { } default: c.AbortWithStatusJSON(http.StatusBadRequest, "missing parameter or payload") + + return } // Add type in message payload ingest.Type = "ingest" From 8e4173d248a20f1228f94bed44f37e0eb8789cf7 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 18 Sep 2025 15:18:38 +0200 Subject: [PATCH 038/184] Rename functions in comments and tests for clarity --- sda/internal/database/db_functions.go | 4 ++-- sda/internal/database/db_functions_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index da653382a..8590f8bd7 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1175,7 +1175,7 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { return accessions, nil } -// GetUserAndPathFromUUID() retrieves user, path and correlation id by giving the file UUID +// GetFileDetailsFromUUID() retrieves user, path and correlation id by giving the file UUID func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { var ( info FileDetails @@ -1193,7 +1193,7 @@ func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { return info, err } -// getUserAndPathFromUUID() is the actual function performing work for GetUserAndPathFromUUID +// getFileDetailsFromUUID() is the actual function performing work for GetUserAndPathFromUUID func (dbs *SDAdb) getFileDetailsFromUUID(fileUUID string) (FileDetails, error) { var info FileDetails dbs.checkAndReconnectIfNeeded() diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index dcc17ce1e..91a2b7427 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1315,7 +1315,7 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { db.Close() } -func (suite *DatabaseTests) TestGetUserAndPathFromUUID_NotFound() { +func (suite *DatabaseTests) TestGetFileDetailsFromUUID_NotFound() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") From 0be5676973cd3e26dbc237cfb7a4695318cb20b1 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:27:54 +0200 Subject: [PATCH 039/184] Apply suggestions from code review Co-authored-by: Nanjiang Shu --- sda/cmd/api/api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index e628dce80..ec5729f01 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -50,7 +50,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `/file/ingest` - accepts `POST` requests with either: - - JSON data: `{"filepath": "", "user": ""}` + - A JSON payload: `{"filepath": "", "user": ""}` - OR a `fileid` query parameter: `/file/ingest?fileid=` - triggers the ingestion of the file. @@ -75,7 +75,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `/file/accession` - accepts `POST` requests with either: - - JSON data: `{"accession_id": "", "filepath": "", "user": ""}` + - A JSON playload: `{"accession_id": "", "filepath": "", "user": ""}` - OR query parameters: `/file/accession?fileid=&accessionid=` - assigns accession ID to the file. From 3f817d6f2083b99c3c1574ff1dc8e88ea9091873 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 25 Sep 2025 16:38:33 +0200 Subject: [PATCH 040/184] Add IngestFileInfo struct and improve PostReq request handling --- sda-admin/helpers/helpers.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index 352ee75b6..da5178ad6 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -12,6 +12,14 @@ import ( // necessary for mocking in unit tests var GetResponseBody = GetBody +type IngestFileInfo struct { + User string + Path string + Id string + Url string + Token string +} + // GetBody sends a GET request to the given URL and returns the body of the response func GetBody(url, token string) ([]byte, error) { req, err := http.NewRequest("GET", url, nil) @@ -50,8 +58,15 @@ var PostRequest = PostReq // PostReq sends a POST request to the server with a JSON body and returns the response body or an error. func PostReq(url, token string, jsonBody []byte) ([]byte, error) { - // Create a new POST request with the provided JSON body - req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBody)) + var req *http.Request + var err error + if jsonBody != nil { + // Create a new POST request with the provided JSON body + req, err = http.NewRequest("POST", url, bytes.NewBuffer(jsonBody)) + } else { + // Create a new POST request with query + req, err = http.NewRequest("POST", url, nil) + } if err != nil { return nil, fmt.Errorf("failed to create the request, reason: %v", err) } From 9910b99d2b4258617c23e2f136d8d265f62f570b Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 25 Sep 2025 16:40:38 +0200 Subject: [PATCH 041/184] Add test case for nil jsonBody in PostReq function --- sda-admin/helpers/helpers_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sda-admin/helpers/helpers_test.go b/sda-admin/helpers/helpers_test.go index 6829d1ecd..9570eabcb 100644 --- a/sda-admin/helpers/helpers_test.go +++ b/sda-admin/helpers/helpers_test.go @@ -64,6 +64,11 @@ func TestPostReq(t *testing.T) { body, err = PostReq(serverError.URL, "mock_token", []byte(`{"name":"test"}`)) assert.Error(t, err) assert.Nil(t, body) + + // Test nil jsonBody case + body, err = PostReq(server.URL, "mock_token", nil) + assert.NoError(t, err) + assert.JSONEq(t, mockResponse, string(body)) } func TestInvalidCharacters(t *testing.T) { From 8b50e72264f4996a7c8a6c87923373b10d8e6eb9 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 25 Sep 2025 16:45:31 +0200 Subject: [PATCH 042/184] Refactor Ingest function - use IngestFileInfo struct - handle ingestion by providing file id as well --- sda-admin/file/file.go | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/sda-admin/file/file.go b/sda-admin/file/file.go index dbb9dc5de..e4890f022 100644 --- a/sda-admin/file/file.go +++ b/sda-admin/file/file.go @@ -39,25 +39,38 @@ func List(apiURI, token, username string) error { return nil } -// Ingest triggers the ingestion of a given file -func Ingest(apiURI, token, username, filepath string) error { - parsedURL, err := url.Parse(apiURI) +// Ingest triggers the ingestion of a file via the SDA API. +// Depending on the provided fields in ingestInfo: +// - If ingestInfo.Id is empty, it sends a POST request to /file/ingest with a JSON body containing the file path and user. +// - If ingestInfo.Id is set, it sends a POST request to /file/ingest with the fileid as a query parameter and no JSON body. +func Ingest(ingestInfo helpers.IngestFileInfo) error { + var jsonBody []byte + parsedURL, err := url.Parse(ingestInfo.Url) if err != nil { return err } parsedURL.Path = path.Join(parsedURL.Path, "file/ingest") - requestBody := RequestBodyFileIngest{ - Filepath: filepath, - User: username, + if ingestInfo.Id == "" { + if err := helpers.CheckValidChars(ingestInfo.Path); err != nil { + return err + } + requestBody := RequestBodyFileIngest{ + Filepath: ingestInfo.Path, + User: ingestInfo.User, + } + jsonBody, err = json.Marshal(requestBody) + if err != nil { + return fmt.Errorf("failed to marshal JSON, reason: %v", err) + } + } else { + query := parsedURL.Query() + query.Set("fileid", ingestInfo.Id) + parsedURL.RawQuery = query.Encode() + jsonBody = nil } - jsonBody, err := json.Marshal(requestBody) - if err != nil { - return fmt.Errorf("failed to marshal JSON, reason: %v", err) - } - - _, err = helpers.PostRequest(parsedURL.String(), token, jsonBody) + _, err = helpers.PostRequest(parsedURL.String(), ingestInfo.Token, jsonBody) if err != nil { return err } From 93f11d50c163cc03187ecd788515b95739622a43 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 25 Sep 2025 17:05:24 +0200 Subject: [PATCH 043/184] Update Ingest unit tests for using struct --- sda-admin/file/file_test.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index 42ad532a9..0da4053f1 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -48,15 +48,17 @@ func TestIngest_Success(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + var ingestInfo helpers.IngestFileInfo expectedURL := "http://example.com/file/ingest" - token := "test-token" - username := "test-user" - filepath := "/path/to/file" + ingestInfo.Url = "http://example.com" + ingestInfo.Token = "test-token" + ingestInfo.User = "test-user" + ingestInfo.Path = "/path/to/file" jsonBody := []byte(`{"filepath":"/path/to/file","user":"test-user"}`) - mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(`{}`), nil) + mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, jsonBody).Return([]byte(`{}`), nil) - err := Ingest("http://example.com", token, username, filepath) + err := Ingest(ingestInfo) assert.NoError(t, err) mockHelpers.AssertExpectations(t) } @@ -67,15 +69,17 @@ func TestIngest_PostRequestFailure(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + var ingestInfo helpers.IngestFileInfo expectedURL := "http://example.com/file/ingest" - token := "test-token" - username := "test-user" - filepath := "/path/to/file" + ingestInfo.Url = "http://example.com" + ingestInfo.Token = "test-token" + ingestInfo.User = "test-user" + ingestInfo.Path = "/path/to/file" jsonBody := []byte(`{"filepath":"/path/to/file","user":"test-user"}`) - mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(nil), errors.New("failed to send request")) + mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, jsonBody).Return([]byte(nil), errors.New("failed to send request")) - err := Ingest("http://example.com", token, username, filepath) + err := Ingest(ingestInfo) assert.Error(t, err) assert.EqualError(t, err, "failed to send request") mockHelpers.AssertExpectations(t) From 3a619824cdf8987fd5a564b2e0afbc8f49067830 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Thu, 25 Sep 2025 17:28:29 +0200 Subject: [PATCH 044/184] Rename Ingest test functions for clarity and add new tests for Ingest with file id functionality --- sda-admin/file/file_test.go | 45 ++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index 0da4053f1..012984009 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -42,7 +42,7 @@ func TestList(t *testing.T) { mockHelpers.AssertExpectations(t) } -func TestIngest_Success(t *testing.T) { +func TestIngestPath_Success(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -63,7 +63,7 @@ func TestIngest_Success(t *testing.T) { mockHelpers.AssertExpectations(t) } -func TestIngest_PostRequestFailure(t *testing.T) { +func TestIngestPath_PostRequestFailure(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -71,7 +71,7 @@ func TestIngest_PostRequestFailure(t *testing.T) { var ingestInfo helpers.IngestFileInfo expectedURL := "http://example.com/file/ingest" - ingestInfo.Url = "http://example.com" + ingestInfo.Url = "http://example.com" ingestInfo.Token = "test-token" ingestInfo.User = "test-user" ingestInfo.Path = "/path/to/file" @@ -85,6 +85,45 @@ func TestIngest_PostRequestFailure(t *testing.T) { mockHelpers.AssertExpectations(t) } +func TestIngestID_Success(t *testing.T) { + mockHelpers := new(MockHelpers) + originalFunc := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + + var ingestInfo helpers.IngestFileInfo + expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" + ingestInfo.Url = "http://example.com" + ingestInfo.Token = "test-token" + ingestInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + + mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, []byte(nil)).Return([]byte(`{}`), nil) + + err := Ingest(ingestInfo) + assert.NoError(t, err) + mockHelpers.AssertExpectations(t) +} + +func TestIngestID_PostRequestFailure(t *testing.T) { + mockHelpers := new(MockHelpers) + originalFunc := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + + var ingestInfo helpers.IngestFileInfo + expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" + ingestInfo.Url = "http://example.com" + ingestInfo.Token = "test-token" + ingestInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + + mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, []byte(nil)).Return([]byte(nil), errors.New("failed to send request")) + + err := Ingest(ingestInfo) + assert.Error(t, err) + assert.EqualError(t, err, "failed to send request") + mockHelpers.AssertExpectations(t) +} + func TestSetAccession_Success(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest From 9d03ae2dc08663bd3c25fec1bee45a2b4640466f Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 10:14:10 +0200 Subject: [PATCH 045/184] Enhance file ingestion command to support both filepath/user and file ID options --- sda-admin/main.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/sda-admin/main.go b/sda-admin/main.go index 62490f6aa..8945cc795 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -76,12 +76,15 @@ var fileListUsage = `Usage: sda-admin file list -user USERNAME Options: -user USERNAME Specify the username associated with the files.` -var fileIngestUsage = `Usage: sda-admin file ingest -filepath FILEPATH -user USERNAME - Trigger the ingestion of a given file for a specific user. +var fileIngestUsage = `Usage with file path and user: sda-admin file ingest -filepath FILEPATH -user USERNAME +Usage with file ID: sda-admin file ingest -fileid FILEUUID + + Trigger the ingestion either by providing filepath and user or file ID. Options: -filepath FILEPATH Specify the path of the file to ingest. - -user USERNAME Specify the username associated with the file.` + -user USERNAME Specify the username associated with the file. + -fileid FILEUUID Specify the file ID (UUID) of the file to ingest.` var fileAccessionUsage = `Usage: sda-admin file set-accession -filepath FILEPATH -user USERNAME -accession-id ACCESSION_ID Assign accession ID to a file and associate it with a user. @@ -336,23 +339,27 @@ func handleFileCommand() error { func handleFileIngestCommand() error { fileIngestCmd := flag.NewFlagSet("ingest", flag.ExitOnError) - var filepath, username string - fileIngestCmd.StringVar(&filepath, "filepath", "", "Filepath to ingest") - fileIngestCmd.StringVar(&username, "user", "", "Username to associate with the file") + var ingestInfo helpers.IngestFileInfo + ingestInfo.Url = apiURI + ingestInfo.Token = token + fileIngestCmd.StringVar(&ingestInfo.Path, "filepath", "", "Filepath to ingest") + fileIngestCmd.StringVar(&ingestInfo.User, "user", "", "Username to associate with the file") + fileIngestCmd.StringVar(&ingestInfo.Id, "fileid", "", "File ID (UUID) to ingest") if err := fileIngestCmd.Parse(flag.Args()[2:]); err != nil { return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) } - if filepath == "" || username == "" { - return fmt.Errorf("error: both -filepath and -user are required.\n%s", fileIngestUsage) - } - - if err := helpers.CheckValidChars(filepath); err != nil { - return err + switch { + case ingestInfo.Path == "" && ingestInfo.User == "" && ingestInfo.Id == "": + return fmt.Errorf("error: either -filepath and -user pair or -fileid are required.\n%s", fileIngestUsage) + case ingestInfo.Id != "" && (ingestInfo.Path != "" || ingestInfo.User != ""): + return fmt.Errorf("error: choose if -filepath and -user pair or -fileid will be used.\n%s", fileIngestUsage) + case ingestInfo.Id == "" && (ingestInfo.Path == "" || ingestInfo.User == ""): + return fmt.Errorf("error: both -filepath and -user must be provided together.\n%s", fileIngestUsage) } - err := file.Ingest(apiURI, token, username, filepath) + err := file.Ingest(ingestInfo) if err != nil { return fmt.Errorf("error: failed to ingest file, reason: %v", err) } From 77ab32244894dcb0d5cae5a517809da63daba3fe Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 13:53:10 +0200 Subject: [PATCH 046/184] Rename and refactor struct to include Accession field --- sda-admin/file/file.go | 2 +- sda-admin/file/file_test.go | 8 ++++---- sda-admin/helpers/helpers.go | 13 +++++++------ 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sda-admin/file/file.go b/sda-admin/file/file.go index e4890f022..6d93f4114 100644 --- a/sda-admin/file/file.go +++ b/sda-admin/file/file.go @@ -43,7 +43,7 @@ func List(apiURI, token, username string) error { // Depending on the provided fields in ingestInfo: // - If ingestInfo.Id is empty, it sends a POST request to /file/ingest with a JSON body containing the file path and user. // - If ingestInfo.Id is set, it sends a POST request to /file/ingest with the fileid as a query parameter and no JSON body. -func Ingest(ingestInfo helpers.IngestFileInfo) error { +func Ingest(ingestInfo helpers.FileInfo) error { var jsonBody []byte parsedURL, err := url.Parse(ingestInfo.Url) if err != nil { diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index 012984009..d650195ee 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -48,7 +48,7 @@ func TestIngestPath_Success(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test - var ingestInfo helpers.IngestFileInfo + var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest" ingestInfo.Url = "http://example.com" ingestInfo.Token = "test-token" @@ -69,7 +69,7 @@ func TestIngestPath_PostRequestFailure(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test - var ingestInfo helpers.IngestFileInfo + var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest" ingestInfo.Url = "http://example.com" ingestInfo.Token = "test-token" @@ -91,7 +91,7 @@ func TestIngestID_Success(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test - var ingestInfo helpers.IngestFileInfo + var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" ingestInfo.Url = "http://example.com" ingestInfo.Token = "test-token" @@ -110,7 +110,7 @@ func TestIngestID_PostRequestFailure(t *testing.T) { helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test - var ingestInfo helpers.IngestFileInfo + var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" ingestInfo.Url = "http://example.com" ingestInfo.Token = "test-token" diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index da5178ad6..27d311008 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -12,12 +12,13 @@ import ( // necessary for mocking in unit tests var GetResponseBody = GetBody -type IngestFileInfo struct { - User string - Path string - Id string - Url string - Token string +type FileInfo struct { + User string + Path string + Id string + Url string + Token string + Accession string } // GetBody sends a GET request to the given URL and returns the body of the response From d1d43571d70c28645332246cc309fca438528c7f Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 14:20:31 +0200 Subject: [PATCH 047/184] Refactor SetAccession function to handle finalize by providing file id --- sda-admin/file/file.go | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/sda-admin/file/file.go b/sda-admin/file/file.go index 6d93f4114..eba35dfcd 100644 --- a/sda-admin/file/file.go +++ b/sda-admin/file/file.go @@ -78,26 +78,40 @@ func Ingest(ingestInfo helpers.FileInfo) error { return nil } -// SetAccession assigns an accession ID to a specified file for a given user -func SetAccession(apiURI, token, username, filepath, accessionID string) error { - parsedURL, err := url.Parse(apiURI) +// SetAccession assigns an accession ID to a file via the SDA API. +// Depending on the provided fields in accessionInfo: +// - If accessionInfo.Id is empty, it sends a POST request to /file/accession with a JSON body containing accession_id, filepath, and user. +// - If accessionInfo.Id is set, it sends a POST request to /file/accession with fileid and accessionid as query parameters. +func SetAccession(accessionInfo helpers.FileInfo) error { + var jsonBody []byte + parsedURL, err := url.Parse(accessionInfo.Url) if err != nil { return err } parsedURL.Path = path.Join(parsedURL.Path, "file/accession") - requestBody := RequestBodyFileAccession{ - AccessionID: accessionID, - Filepath: filepath, - User: username, - } - - jsonBody, err := json.Marshal(requestBody) - if err != nil { - return fmt.Errorf("failed to marshal JSON, reason: %v", err) + if accessionInfo.Id == "" { + if err := helpers.CheckValidChars(accessionInfo.Path); err != nil { + return err + } + requestBody := RequestBodyFileAccession{ + AccessionID: accessionInfo.Accession, + Filepath: accessionInfo.Path, + User: accessionInfo.User, + } + jsonBody, err = json.Marshal(requestBody) + if err != nil { + return fmt.Errorf("failed to marshal JSON, reason: %v", err) + } + } else { + query := parsedURL.Query() + query.Set("fileid", accessionInfo.Id) + query.Set("accessionid", accessionInfo.Accession) + parsedURL.RawQuery = query.Encode() + jsonBody = nil } - _, err = helpers.PostRequest(parsedURL.String(), token, jsonBody) + _, err = helpers.PostRequest(parsedURL.String(), accessionInfo.Token, jsonBody) if err != nil { return err } From d11f4c2c3cdf3a11d15e44cfc8962ad8222dadbb Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 15:21:51 +0200 Subject: [PATCH 048/184] Update SetAccession unit tests for using FileInfo struct --- sda-admin/file/file_test.go | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index d650195ee..5e21235b1 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -124,42 +124,46 @@ func TestIngestID_PostRequestFailure(t *testing.T) { mockHelpers.AssertExpectations(t) } -func TestSetAccession_Success(t *testing.T) { +func TestSetAccessionPath_Success(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession" - token := "test-token" - username := "test-user" - filepath := "/path/to/file" - accessionID := "accession-123" + accessionInfo.Url = "http://example.com" + accessionInfo.Token = "test-token" + accessionInfo.User = "test-user" + accessionInfo.Path = "/path/to/file" + accessionInfo.Accession = "accession-123" jsonBody := []byte(`{"accession_id":"accession-123","filepath":"/path/to/file","user":"test-user"}`) - mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(`{}`), nil) + mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, jsonBody).Return([]byte(`{}`), nil) - err := SetAccession("http://example.com", token, username, filepath, accessionID) + err := SetAccession(accessionInfo) assert.NoError(t, err) mockHelpers.AssertExpectations(t) } -func TestSetAccession_PostRequestFailure(t *testing.T) { +func TestSetAccessionPath_PostRequestFailure(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession" - token := "test-token" - username := "test-user" - filepath := "/path/to/file" - accessionID := "accession-123" + accessionInfo.Url = "http://example.com" + accessionInfo.Token = "test-token" + accessionInfo.User = "test-user" + accessionInfo.Path = "/path/to/file" + accessionInfo.Accession = "accession-123" jsonBody := []byte(`{"accession_id":"accession-123","filepath":"/path/to/file","user":"test-user"}`) - mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(nil), errors.New("failed to send request")) + mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, jsonBody).Return([]byte(nil), errors.New("failed to send request")) - err := SetAccession("http://example.com", token, username, filepath, accessionID) + err := SetAccession(accessionInfo) assert.Error(t, err) assert.EqualError(t, err, "failed to send request") mockHelpers.AssertExpectations(t) From 7fe7b89b98d7e829de513c6ff2cfd955fc4b8a76 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 16:07:45 +0200 Subject: [PATCH 049/184] Add new unit tests for SetAccession with file id functionality --- sda-admin/file/file_test.go | 41 +++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index 5e21235b1..db9835fa8 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -168,3 +168,44 @@ func TestSetAccessionPath_PostRequestFailure(t *testing.T) { assert.EqualError(t, err, "failed to send request") mockHelpers.AssertExpectations(t) } + +func TestSetAccessionID_Success(t *testing.T) { + mockHelpers := new(MockHelpers) + originalFunc := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + + var accessionInfo helpers.FileInfo + expectedURL := "http://example.com/file/accession?accessionid=accession-123&fileid=dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.Url = "http://example.com" + accessionInfo.Token = "test-token" + accessionInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.Accession = "accession-123" + + mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, []byte(nil)).Return([]byte(`{}`), nil) + + err := SetAccession(accessionInfo) + assert.NoError(t, err) + mockHelpers.AssertExpectations(t) +} + +func TestSetAccessionID_PostRequestFailure(t *testing.T) { + mockHelpers := new(MockHelpers) + originalFunc := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalFunc }() // Restore original after test + + var accessionInfo helpers.FileInfo + expectedURL := "http://example.com/file/accession?accessionid=accession-123&fileid=dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.Url = "http://example.com" + accessionInfo.Token = "test-token" + accessionInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.Accession = "accession-123" + + mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, []byte(nil)).Return([]byte(nil), errors.New("failed to send request")) + + err := SetAccession(accessionInfo) + assert.Error(t, err) + assert.EqualError(t, err, "failed to send request") + mockHelpers.AssertExpectations(t) +} From 4267727d70413c5303343a96001c73d3fa2c2269 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 16:09:37 +0200 Subject: [PATCH 050/184] Enhance file set-accession command to support both filepath/user and file ID options --- sda-admin/main.go | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/sda-admin/main.go b/sda-admin/main.go index 8945cc795..7dd11958b 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -86,12 +86,15 @@ Options: -user USERNAME Specify the username associated with the file. -fileid FILEUUID Specify the file ID (UUID) of the file to ingest.` -var fileAccessionUsage = `Usage: sda-admin file set-accession -filepath FILEPATH -user USERNAME -accession-id ACCESSION_ID - Assign accession ID to a file and associate it with a user. +var fileAccessionUsage = `Usage with file path and user: sda-admin file set-accession -filepath FILEPATH -user USERNAME -accession-id ACCESSION_ID +Usage with file ID: sda-admin file set-accession -fileid FILEUUID -accession-id ACCESSION_ID + + Assign accession ID to a file by providing filepath and user or file ID. Options: -filepath FILEPATH Specify the path of the file to assign the accession ID. -user USERNAME Specify the username associated with the file. + -fileid FILEUUID Specify the file ID of the file to assign the accession ID. -accession-id ID Specify the accession ID to assign to the file.` var datasetUsage = `Create a dataset: @@ -339,7 +342,7 @@ func handleFileCommand() error { func handleFileIngestCommand() error { fileIngestCmd := flag.NewFlagSet("ingest", flag.ExitOnError) - var ingestInfo helpers.IngestFileInfo + var ingestInfo helpers.FileInfo ingestInfo.Url = apiURI ingestInfo.Token = token fileIngestCmd.StringVar(&ingestInfo.Path, "filepath", "", "Filepath to ingest") @@ -369,24 +372,32 @@ func handleFileIngestCommand() error { func handleFileAccessionCommand() error { fileAccessionCmd := flag.NewFlagSet("set-accession", flag.ExitOnError) - var filepath, username, accessionID string - fileAccessionCmd.StringVar(&filepath, "filepath", "", "Filepath to assign accession ID") - fileAccessionCmd.StringVar(&username, "user", "", "Username to associate with the file") - fileAccessionCmd.StringVar(&accessionID, "accession-id", "", "Accession ID to assign") + var accessionInfo helpers.FileInfo + accessionInfo.Url = apiURI + accessionInfo.Token = token + fileAccessionCmd.StringVar(&accessionInfo.Path, "filepath", "", "Filepath to assign accession ID") + fileAccessionCmd.StringVar(&accessionInfo.User, "user", "", "Username to associate with the file") + fileAccessionCmd.StringVar(&accessionInfo.Accession, "accession-id", "", "Accession ID to assign") + fileAccessionCmd.StringVar(&accessionInfo.Id, "fileid", "", "File ID (UUID) to ingest") if err := fileAccessionCmd.Parse(flag.Args()[2:]); err != nil { return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) } - if filepath == "" || username == "" || accessionID == "" { + switch { + case accessionInfo.Id == "" && accessionInfo.Path == "" && accessionInfo.User == "" && accessionInfo.Accession == "": + return fmt.Errorf("error: no arguments provided.\n%s", fileAccessionUsage) + case accessionInfo.Id == "" && (accessionInfo.Path == "" || accessionInfo.User == "" || accessionInfo.Accession == ""): return fmt.Errorf("error: -filepath, -user, and -accession-id are required.\n%s", fileAccessionUsage) + case accessionInfo.Id != "" && accessionInfo.Accession != "" && (accessionInfo.Path != "" || accessionInfo.User != ""): + return fmt.Errorf("error: when using -fileid, do not provide -filepath or -user together. Only -fileid and -accession-id are allowed.\n%s", fileAccessionUsage) + case accessionInfo.Id != "" && accessionInfo.Accession == "" && (accessionInfo.Path == "" && accessionInfo.User == ""): + return fmt.Errorf("error: -accession-id is required.\n%s", fileAccessionUsage) + case accessionInfo.Id == "" && accessionInfo.Path != "" && accessionInfo.User != "" && accessionInfo.Accession == "": + return fmt.Errorf("error: -accession-id is required.\n%s", fileAccessionUsage) } - if err := helpers.CheckValidChars(filepath); err != nil { - return err - } - - err := file.SetAccession(apiURI, token, username, filepath, accessionID) + err := file.SetAccession(accessionInfo) if err != nil { return fmt.Errorf("error: failed to assign accession ID to file, reason: %v", err) } From 621d7a1b754573e77a41d8c34d907fbacdfccacb Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 26 Sep 2025 16:14:30 +0200 Subject: [PATCH 051/184] Update README --- sda-admin/README.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sda-admin/README.md b/sda-admin/README.md index fe9851036..fd8bbf083 100644 --- a/sda-admin/README.md +++ b/sda-admin/README.md @@ -28,22 +28,35 @@ Use the following command to return all files belonging to the specified user `t sda-admin file list -user test-user@example.org ``` + ## Ingest a file -Use the following command to trigger the ingesting of a given file `/path/to/file.c4gh` that belongs to the user `test-user@example.org` +You can ingest a file either by specifying its path and user, or by using its file ID: +**By file path and user:** ```sh sda-admin file ingest -filepath /path/to/file.c4gh -user test-user@example.org ``` +**By file ID:** +```sh +sda-admin file ingest -fileid +``` + ## Assign an accession ID to a file -Use the following command to assign an accession ID `my-accession-id-1` to a given file `/path/to/file.c4gh` that belongs to the user `test-user@example.org` +You can assign an accession ID to a file either by specifying its path and user, or by using its file ID: +**By file path and user:** ```sh sda-admin file set-accession -filepath /path/to/file.c4gh -user test-user@example.org -accession-id my-accession-id-1 ``` +**By file ID:** +```sh +sda-admin file set-accession -fileid -accession-id my-accession-id-1 +``` + ## Create a dataset from a list of accession IDs and a dataset ID Use the following command to create a dataset `dataset001` from accession IDs `my-accession-id-1` and `my-accession-id-2` for files that belongs to the user `test-user@example.org` From ffb95077b332bdc0d932ae19ba90d6c896e6ef6a Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Thu, 9 Oct 2025 13:45:57 +0200 Subject: [PATCH 052/184] Update .github/integration/tests/sda/60_api_admin_test.sh Co-authored-by: Nanjiang Shu --- .github/integration/tests/sda/60_api_admin_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index a81b7be93..52099804d 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -245,7 +245,7 @@ echo "Ingestion by using file ID finished successfully" # Test giving accession id to a file by using file id echo "Giving accession id by using file id" # The file which ingested above will be used -accession_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/file/accession?fileid=$new_fileid&accessionid=SDA-123-asd")" +accession_resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X POST "http://api:8080/file/accession?fileid=$new_fileid&accessionid=SDA-123-asd")" if [ "$accession_resp" != "200" ]; then echo "Error when requesting to finalize file by the use of file id, expected 200 got: $accession_resp" exit 1 From d311418f226fd78d6e27d8a887cb68e5971dbbc4 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Fri, 10 Oct 2025 11:03:09 +0200 Subject: [PATCH 053/184] Linter fixes --- sda-admin/file/file.go | 12 ++++----- sda-admin/file/file_test.go | 24 +++++++++--------- sda-admin/helpers/helpers.go | 4 +-- sda-admin/main.go | 48 ++++++++++++++++++------------------ 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/sda-admin/file/file.go b/sda-admin/file/file.go index eba35dfcd..e84cd5d85 100644 --- a/sda-admin/file/file.go +++ b/sda-admin/file/file.go @@ -45,13 +45,13 @@ func List(apiURI, token, username string) error { // - If ingestInfo.Id is set, it sends a POST request to /file/ingest with the fileid as a query parameter and no JSON body. func Ingest(ingestInfo helpers.FileInfo) error { var jsonBody []byte - parsedURL, err := url.Parse(ingestInfo.Url) + parsedURL, err := url.Parse(ingestInfo.URL) if err != nil { return err } parsedURL.Path = path.Join(parsedURL.Path, "file/ingest") - if ingestInfo.Id == "" { + if ingestInfo.ID == "" { if err := helpers.CheckValidChars(ingestInfo.Path); err != nil { return err } @@ -65,7 +65,7 @@ func Ingest(ingestInfo helpers.FileInfo) error { } } else { query := parsedURL.Query() - query.Set("fileid", ingestInfo.Id) + query.Set("fileid", ingestInfo.ID) parsedURL.RawQuery = query.Encode() jsonBody = nil } @@ -84,13 +84,13 @@ func Ingest(ingestInfo helpers.FileInfo) error { // - If accessionInfo.Id is set, it sends a POST request to /file/accession with fileid and accessionid as query parameters. func SetAccession(accessionInfo helpers.FileInfo) error { var jsonBody []byte - parsedURL, err := url.Parse(accessionInfo.Url) + parsedURL, err := url.Parse(accessionInfo.URL) if err != nil { return err } parsedURL.Path = path.Join(parsedURL.Path, "file/accession") - if accessionInfo.Id == "" { + if accessionInfo.ID == "" { if err := helpers.CheckValidChars(accessionInfo.Path); err != nil { return err } @@ -105,7 +105,7 @@ func SetAccession(accessionInfo helpers.FileInfo) error { } } else { query := parsedURL.Query() - query.Set("fileid", accessionInfo.Id) + query.Set("fileid", accessionInfo.ID) query.Set("accessionid", accessionInfo.Accession) parsedURL.RawQuery = query.Encode() jsonBody = nil diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index db9835fa8..c14bf519f 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -50,7 +50,7 @@ func TestIngestPath_Success(t *testing.T) { var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest" - ingestInfo.Url = "http://example.com" + ingestInfo.URL = "http://example.com" ingestInfo.Token = "test-token" ingestInfo.User = "test-user" ingestInfo.Path = "/path/to/file" @@ -71,7 +71,7 @@ func TestIngestPath_PostRequestFailure(t *testing.T) { var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest" - ingestInfo.Url = "http://example.com" + ingestInfo.URL = "http://example.com" ingestInfo.Token = "test-token" ingestInfo.User = "test-user" ingestInfo.Path = "/path/to/file" @@ -93,9 +93,9 @@ func TestIngestID_Success(t *testing.T) { var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" - ingestInfo.Url = "http://example.com" + ingestInfo.URL = "http://example.com" ingestInfo.Token = "test-token" - ingestInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + ingestInfo.ID = "dd813b8a-ea90-4556-b640-32039733a31f" mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, []byte(nil)).Return([]byte(`{}`), nil) @@ -112,9 +112,9 @@ func TestIngestID_PostRequestFailure(t *testing.T) { var ingestInfo helpers.FileInfo expectedURL := "http://example.com/file/ingest?fileid=dd813b8a-ea90-4556-b640-32039733a31f" - ingestInfo.Url = "http://example.com" + ingestInfo.URL = "http://example.com" ingestInfo.Token = "test-token" - ingestInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + ingestInfo.ID = "dd813b8a-ea90-4556-b640-32039733a31f" mockHelpers.On("PostRequest", expectedURL, ingestInfo.Token, []byte(nil)).Return([]byte(nil), errors.New("failed to send request")) @@ -132,7 +132,7 @@ func TestSetAccessionPath_Success(t *testing.T) { var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession" - accessionInfo.Url = "http://example.com" + accessionInfo.URL = "http://example.com" accessionInfo.Token = "test-token" accessionInfo.User = "test-user" accessionInfo.Path = "/path/to/file" @@ -154,7 +154,7 @@ func TestSetAccessionPath_PostRequestFailure(t *testing.T) { var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession" - accessionInfo.Url = "http://example.com" + accessionInfo.URL = "http://example.com" accessionInfo.Token = "test-token" accessionInfo.User = "test-user" accessionInfo.Path = "/path/to/file" @@ -177,9 +177,9 @@ func TestSetAccessionID_Success(t *testing.T) { var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession?accessionid=accession-123&fileid=dd813b8a-ea90-4556-b640-32039733a31f" - accessionInfo.Url = "http://example.com" + accessionInfo.URL = "http://example.com" accessionInfo.Token = "test-token" - accessionInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.ID = "dd813b8a-ea90-4556-b640-32039733a31f" accessionInfo.Accession = "accession-123" mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, []byte(nil)).Return([]byte(`{}`), nil) @@ -197,9 +197,9 @@ func TestSetAccessionID_PostRequestFailure(t *testing.T) { var accessionInfo helpers.FileInfo expectedURL := "http://example.com/file/accession?accessionid=accession-123&fileid=dd813b8a-ea90-4556-b640-32039733a31f" - accessionInfo.Url = "http://example.com" + accessionInfo.URL = "http://example.com" accessionInfo.Token = "test-token" - accessionInfo.Id = "dd813b8a-ea90-4556-b640-32039733a31f" + accessionInfo.ID = "dd813b8a-ea90-4556-b640-32039733a31f" accessionInfo.Accession = "accession-123" mockHelpers.On("PostRequest", expectedURL, accessionInfo.Token, []byte(nil)).Return([]byte(nil), errors.New("failed to send request")) diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index 27d311008..0fc70af29 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -15,8 +15,8 @@ var GetResponseBody = GetBody type FileInfo struct { User string Path string - Id string - Url string + ID string + URL string Token string Accession string } diff --git a/sda-admin/main.go b/sda-admin/main.go index 7dd11958b..8cd4b56ed 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -343,66 +343,66 @@ func handleFileCommand() error { func handleFileIngestCommand() error { fileIngestCmd := flag.NewFlagSet("ingest", flag.ExitOnError) var ingestInfo helpers.FileInfo - ingestInfo.Url = apiURI + ingestInfo.URL = apiURI ingestInfo.Token = token fileIngestCmd.StringVar(&ingestInfo.Path, "filepath", "", "Filepath to ingest") fileIngestCmd.StringVar(&ingestInfo.User, "user", "", "Username to associate with the file") - fileIngestCmd.StringVar(&ingestInfo.Id, "fileid", "", "File ID (UUID) to ingest") + fileIngestCmd.StringVar(&ingestInfo.ID, "fileid", "", "File ID (UUID) to ingest") if err := fileIngestCmd.Parse(flag.Args()[2:]); err != nil { return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) } switch { - case ingestInfo.Path == "" && ingestInfo.User == "" && ingestInfo.Id == "": + case ingestInfo.Path == "" && ingestInfo.User == "" && ingestInfo.ID == "": return fmt.Errorf("error: either -filepath and -user pair or -fileid are required.\n%s", fileIngestUsage) - case ingestInfo.Id != "" && (ingestInfo.Path != "" || ingestInfo.User != ""): + case ingestInfo.ID != "" && (ingestInfo.Path != "" || ingestInfo.User != ""): return fmt.Errorf("error: choose if -filepath and -user pair or -fileid will be used.\n%s", fileIngestUsage) - case ingestInfo.Id == "" && (ingestInfo.Path == "" || ingestInfo.User == ""): + case ingestInfo.ID == "" && (ingestInfo.Path == "" || ingestInfo.User == ""): return fmt.Errorf("error: both -filepath and -user must be provided together.\n%s", fileIngestUsage) - } + default: + err := file.Ingest(ingestInfo) + if err != nil { + return fmt.Errorf("error: failed to ingest file, reason: %v", err) + } - err := file.Ingest(ingestInfo) - if err != nil { - return fmt.Errorf("error: failed to ingest file, reason: %v", err) + return nil } - - return nil } func handleFileAccessionCommand() error { fileAccessionCmd := flag.NewFlagSet("set-accession", flag.ExitOnError) var accessionInfo helpers.FileInfo - accessionInfo.Url = apiURI + accessionInfo.URL = apiURI accessionInfo.Token = token fileAccessionCmd.StringVar(&accessionInfo.Path, "filepath", "", "Filepath to assign accession ID") fileAccessionCmd.StringVar(&accessionInfo.User, "user", "", "Username to associate with the file") fileAccessionCmd.StringVar(&accessionInfo.Accession, "accession-id", "", "Accession ID to assign") - fileAccessionCmd.StringVar(&accessionInfo.Id, "fileid", "", "File ID (UUID) to ingest") + fileAccessionCmd.StringVar(&accessionInfo.ID, "fileid", "", "File ID (UUID) to ingest") if err := fileAccessionCmd.Parse(flag.Args()[2:]); err != nil { return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) } switch { - case accessionInfo.Id == "" && accessionInfo.Path == "" && accessionInfo.User == "" && accessionInfo.Accession == "": + case accessionInfo.ID == "" && accessionInfo.Path == "" && accessionInfo.User == "" && accessionInfo.Accession == "": return fmt.Errorf("error: no arguments provided.\n%s", fileAccessionUsage) - case accessionInfo.Id == "" && (accessionInfo.Path == "" || accessionInfo.User == "" || accessionInfo.Accession == ""): + case accessionInfo.ID == "" && (accessionInfo.Path == "" || accessionInfo.User == "" || accessionInfo.Accession == ""): return fmt.Errorf("error: -filepath, -user, and -accession-id are required.\n%s", fileAccessionUsage) - case accessionInfo.Id != "" && accessionInfo.Accession != "" && (accessionInfo.Path != "" || accessionInfo.User != ""): + case accessionInfo.ID != "" && accessionInfo.Accession != "" && (accessionInfo.Path != "" || accessionInfo.User != ""): return fmt.Errorf("error: when using -fileid, do not provide -filepath or -user together. Only -fileid and -accession-id are allowed.\n%s", fileAccessionUsage) - case accessionInfo.Id != "" && accessionInfo.Accession == "" && (accessionInfo.Path == "" && accessionInfo.User == ""): + case accessionInfo.ID != "" && accessionInfo.Accession == "" && (accessionInfo.Path == "" && accessionInfo.User == ""): return fmt.Errorf("error: -accession-id is required.\n%s", fileAccessionUsage) - case accessionInfo.Id == "" && accessionInfo.Path != "" && accessionInfo.User != "" && accessionInfo.Accession == "": + case accessionInfo.ID == "" && accessionInfo.Path != "" && accessionInfo.User != "" && accessionInfo.Accession == "": return fmt.Errorf("error: -accession-id is required.\n%s", fileAccessionUsage) - } + default: + err := file.SetAccession(accessionInfo) + if err != nil { + return fmt.Errorf("error: failed to assign accession ID to file, reason: %v", err) + } - err := file.SetAccession(accessionInfo) - if err != nil { - return fmt.Errorf("error: failed to assign accession ID to file, reason: %v", err) + return nil } - - return nil } func handleDatasetCommand() error { From 54dc1535e666793b34acb0abaec469bf8864040d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 17:06:16 +0000 Subject: [PATCH 054/184] Bump github.com/quic-go/quic-go Bumps the go_modules group with 1 update in the /sda-download directory: [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go). Updates `github.com/quic-go/quic-go` from 0.54.0 to 0.54.1 - [Release notes](https://github.com/quic-go/quic-go/releases) - [Commits](https://github.com/quic-go/quic-go/compare/v0.54.0...v0.54.1) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-version: 0.54.1 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 2a0767d48..92ff33eab 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -56,7 +56,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/quic-go v0.54.1 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/segmentio/asm v1.2.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 59f3f2b69..f32759ca6 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -113,8 +113,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= -github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= From d4c03921a0f0653a8a0edf99b92a10342edbfbb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Oct 2025 17:07:36 +0000 Subject: [PATCH 055/184] Bump github.com/quic-go/quic-go Bumps the go_modules group with 1 update in the /sda directory: [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go). Updates `github.com/quic-go/quic-go` from 0.54.0 to 0.54.1 - [Release notes](https://github.com/quic-go/quic-go/releases) - [Commits](https://github.com/quic-go/quic-go/compare/v0.54.0...v0.54.1) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-version: 0.54.1 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 50208bfe4..53d8a82e0 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -134,7 +134,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/quic-go/qpack v0.5.1 // indirect - github.com/quic-go/quic-go v0.54.0 // indirect + github.com/quic-go/quic-go v0.54.1 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index c83d74b85..ffac05408 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -303,8 +303,8 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= -github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= -github.com/quic-go/quic-go v0.54.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= +github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg= +github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= From ec010653ee899ae73b14b4aeee1c9285da2f3309 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Sat, 11 Oct 2025 18:06:10 +0200 Subject: [PATCH 056/184] Fix finalize with json payload --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 1d16b992e..da9586ab4 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -631,7 +631,7 @@ func setAccession(c *gin.Context) { return } // Find the correlation id - fileID, err := Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "uploaded") + fileID, err := Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "verified") if err != nil { if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) From 2e4592b14f6f40b05e424c7812d3cc26f0f1156b Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Sat, 11 Oct 2025 23:49:49 +0200 Subject: [PATCH 057/184] Add api integration test for finalize (json payload) --- .../tests/sda/60_api_admin_test.sh | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 52099804d..d4340c05e 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -147,6 +147,35 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e sleep 2 done +# Finalize file +echo "Giving accession id by using json payload" +accession_payload=$( +jq -c -n \ + --arg filepath "NE12878.bam.c4gh" \ + --arg user "test@dummy.org" \ + --arg accession_id "my-id-01" \ + '$ARGS.named' +) + +resp_accession_payload="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$accession_payload" "http://api:8080/file/accession")" +if [ "$resp_accession_payload" != "200" ]; then + echo "Error when requesting to ingesting file, expected 200 got: $resp_accession_payload" + exit 1 +fi + +# Check that the file has been finalized +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$fileid' order by started_at desc limit 1;")" = "ready" ]; do + echo "waiting for finalize to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 10 ]; then + echo "::error::Time out while waiting for finalizing to complete" + exit 1 + fi + sleep 2 +done +echo "Finalize by using json payload finished successfully" + # Try to delete file not in inbox fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')" resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" From a9a0c7dfd27f37b4a6255f341cf7322b55b8b93b Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Mon, 13 Oct 2025 10:56:45 +0200 Subject: [PATCH 058/184] Refactor GetFileDetailsFromUUID to accept event type and update related calls --- sda/cmd/api/api.go | 4 ++-- sda/cmd/api/api_test.go | 2 ++ sda/internal/database/db_functions.go | 10 +++++----- sda/internal/database/db_functions_test.go | 4 ++-- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index da9586ab4..1c4f38908 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -337,7 +337,7 @@ func ingestFile(c *gin.Context) { return case c.Query("fileid") != "": // Get the user and the inbox filepath - fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) + fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid"), "uploaded") if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, "file information not found") @@ -597,7 +597,7 @@ func setAccession(c *gin.Context) { return case c.Query("fileid") != "" && c.Query("accessionid") != "": // Get the user and the inbox filepath - fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid")) + fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid"), "verified") if err != nil { c.AbortWithStatusJSON(http.StatusBadRequest, "file details not found") diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 49d4f967e..cd8660b1b 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -323,6 +323,8 @@ func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Verified") + err = Conf.API.DB.UpdateFileEventLog(fileID, "verified", fileID, user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update status of file in database") return fileID, decSha } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 8590f8bd7..a14bb4337 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1176,14 +1176,14 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { } // GetFileDetailsFromUUID() retrieves user, path and correlation id by giving the file UUID -func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { +func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID, event string) (FileDetails, error) { var ( info FileDetails err error ) for count := 0; count <= RetryTimes; count++ { - info, err = dbs.getFileDetailsFromUUID(fileUUID) + info, err = dbs.getFileDetailsFromUUID(fileUUID, event) if err == nil { break } @@ -1194,15 +1194,15 @@ func (dbs *SDAdb) GetFileDetailsFromUUID(fileUUID string) (FileDetails, error) { } // getFileDetailsFromUUID() is the actual function performing work for GetUserAndPathFromUUID -func (dbs *SDAdb) getFileDetailsFromUUID(fileUUID string) (FileDetails, error) { +func (dbs *SDAdb) getFileDetailsFromUUID(fileUUID, event string) (FileDetails, error) { var info FileDetails dbs.checkAndReconnectIfNeeded() const query = `SELECT f.submission_user, f.submission_file_path, fel.correlation_id from sda.files f join sda.file_event_log fel on f.id = fel.file_id - WHERE f.id = $1 and fel.event='uploaded';` - if err := dbs.DB.QueryRow(query, fileUUID).Scan(&info.User, &info.Path, &info.CorrID); err != nil { + WHERE f.id = $1 and fel.event=$2;` + if err := dbs.DB.QueryRow(query, fileUUID, event).Scan(&info.User, &info.Path, &info.CorrID); err != nil { return FileDetails{}, err } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 91a2b7427..149329d4b 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1307,7 +1307,7 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { suite.FailNow("failed to update file event log") } - infoFile, err := db.GetFileDetailsFromUUID(fileID) + infoFile, err := db.GetFileDetailsFromUUID(fileID, "uploaded") assert.NoError(suite.T(), err, "failed to get user and path from UUID") assert.Equal(suite.T(), user, infoFile.User) assert.Equal(suite.T(), filePath, infoFile.Path) @@ -1321,7 +1321,7 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUID_NotFound() { // Use a non-existent UUID invalidUUID := "abc-123" - infoFile, err := db.GetFileDetailsFromUUID(invalidUUID) + infoFile, err := db.GetFileDetailsFromUUID(invalidUUID, "uploaded") assert.Error(suite.T(), err, "expected error for non-existent UUID") assert.Empty(suite.T(), infoFile.User) assert.Empty(suite.T(), infoFile.Path) From a9f92913ac91b1ca0a04d937aa89361ca814b8a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 19:08:29 +0000 Subject: [PATCH 059/184] Bump github/codeql-action from 3 to 4 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3 to 4. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v3...v4) --- updated-dependencies: - dependency-name: github/codeql-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 6 +++--- .github/workflows/codeql-analysis.yml | 6 +++--- .github/workflows/scan-images.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 813b943d9..be22308a9 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -117,7 +117,7 @@ jobs: output: 'postgres-results.sarif' severity: "CRITICAL,HIGH" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: 'postgres-results.sarif' category: postgres @@ -135,7 +135,7 @@ jobs: output: 'rabbitmq-results.sarif' severity: "CRITICAL,HIGH" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: 'rabbitmq-results.sarif' category: rabbitmq @@ -184,7 +184,7 @@ jobs: output: 'inbox-results.sarif' severity: "CRITICAL,HIGH" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: 'inbox-results.sarif' category: sftp-inbox diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d2cd99f94..6080a5951 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,7 +29,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} @@ -57,9 +57,9 @@ jobs: - name: Autobuild if: ${{ matrix.language == 'go' }} - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index c1ea49fdc..f764e13f2 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -39,7 +39,7 @@ jobs: output: '${{ matrix.image-name }}-results.sarif' severity: "CRITICAL,HIGH" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: '${{ matrix.image-name }}-results.sarif' category: ${{ matrix.image-name }} @@ -65,7 +65,7 @@ jobs: output: 'sda-results.sarif' severity: "CRITICAL,HIGH" - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: 'sda-results.sarif' category: sda \ No newline at end of file From 5a1a10fb2771061ffb631c1f48d1609723f572f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 19:16:04 +0000 Subject: [PATCH 060/184] Bump com.squareup.okhttp3:okhttp-jvm Bumps the all-modules group in /sda-doa with 1 update: [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp). Updates `com.squareup.okhttp3:okhttp-jvm` from 5.1.0 to 5.2.1 - [Changelog](https://github.com/square/okhttp/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okhttp/compare/parent-5.1.0...parent-5.2.1) --- updated-dependencies: - dependency-name: com.squareup.okhttp3:okhttp-jvm dependency-version: 5.2.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 791c4cf1e..f58971cd0 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -137,7 +137,7 @@ com.squareup.okhttp3 okhttp-jvm - 5.1.0 + 5.2.1 From 108fc7c61d3d25fe79ea58a38df617b46d0ef62c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Oct 2025 12:36:30 +0000 Subject: [PATCH 061/184] Bump the all-modules group across 1 directory with 3 updates Bumps the all-modules group with 3 updates in the /sda-download directory: [golang.org/x/crypto](https://github.com/golang/crypto), [google.golang.org/grpc](https://github.com/grpc/grpc-go) and google.golang.org/protobuf. Updates `golang.org/x/crypto` from 0.42.0 to 0.43.0 - [Commits](https://github.com/golang/crypto/compare/v0.42.0...v0.43.0) Updates `google.golang.org/grpc` from 1.75.1 to 1.76.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.75.1...v1.76.0) Updates `google.golang.org/protobuf` from 1.36.9 to 1.36.10 --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.43.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.76.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-version: 1.36.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 18 +++++++++--------- sda-download/go.sum | 40 ++++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 92ff33eab..b5e36f4b9 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,10 +15,10 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.42.0 + golang.org/x/crypto v0.43.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.75.1 - google.golang.org/protobuf v1.36.9 + google.golang.org/grpc v1.76.0 + google.golang.org/protobuf v1.36.10 ) require ( @@ -71,13 +71,13 @@ require ( go.uber.org/mock v0.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.43.0 // indirect + golang.org/x/mod v0.28.0 // indirect + golang.org/x/net v0.45.0 // indirect golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.29.0 // indirect - golang.org/x/tools v0.36.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/tools v0.37.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index f32759ca6..111583084 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -179,23 +179,23 @@ golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= +golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -212,40 +212,40 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= -google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From eeb6a450410676afca30f1595fb8dad9bf8c4653 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Mon, 20 Oct 2025 16:45:55 +0200 Subject: [PATCH 062/184] Update sda/cmd/api/api.go Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 1c4f38908..0b2d76ba0 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -607,7 +607,7 @@ func setAccession(c *gin.Context) { fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(c.Query("fileid")) if err != nil { log.Debugln(err.Error()) - c.AbortWithStatusJSON(http.StatusNotFound, "decrypted checksum not found") + c.AbortWithStatusJSON(http.StatusInternalServerError, "required data missing") return } From a1a033c9a8e887843ca0e2e6f729ed5ff6017e31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 19:14:25 +0000 Subject: [PATCH 063/184] Bump no.elixir:crypt4gh in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.34 to 3.0.35 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.34...crypt4gh-3.0.35) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.35 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index f58971cd0..969205b6a 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.34 + 3.0.35 org.slf4j From b034cb324ca7a62eab9675576610e521fdb597af Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 00:50:48 +0200 Subject: [PATCH 064/184] make GetC4GHPublicKey work for different apps --- sda/cmd/sync/sync.go | 2 +- sda/internal/config/config.go | 17 ++++++++++++++--- sda/internal/config/config_test.go | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 7d71051a7..b8f2fb638 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -60,7 +60,7 @@ func main() { log.Fatal(err) } - publicKey, err = config.GetC4GHPublicKey() + publicKey, err = config.GetC4GHPublicKey("sync") if err != nil { log.Fatal(err) } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 941c2b265..c0a715cb7 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -713,6 +713,7 @@ func NewConfig(app string) (*Config, error) { return c, nil } +// configAPI provides configuration for the api web server func (c *Config) configAPI() error { c.apiDefaults() api := APIConf{} @@ -1130,7 +1131,7 @@ func (c *Config) configSync() error { return nil } -// configSync provides configuration for the outgoing sync settings +// configSyncAPI provides configuration for the outgoing sync settings func (c *Config) configSyncAPI() { c.SyncAPI = SyncAPIConf{} c.SyncAPI.APIPassword = viper.GetString("sync.api.password") @@ -1197,8 +1198,18 @@ func GetC4GHprivateKeys() ([]*[32]byte, error) { } // GetC4GHPublicKey reads the c4gh public key -func GetC4GHPublicKey() (*[32]byte, error) { - keyPath := viper.GetString("c4gh.syncPubKeyPath") +func GetC4GHPublicKey(app string) (*[32]byte, error) { + + var keyPath string + switch app { + case "sync": + keyPath = viper.GetString("c4gh.syncPubKeyPath") + case "rotatekey": + keyPath = viper.GetString("c4gh.rotatePubKeyPath") + default: + return nil, errors.New("pubKey not set") + } + // Make sure the key path and passphrase is valid keyFile, err := os.Open(keyPath) if err != nil { diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 4cde11881..f58b4139d 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -337,7 +337,7 @@ func (ts *ConfigTestSuite) TestGetC4GHPublicKey() { copy(kb[:], k) viper.Set("c4gh.syncPubKeyPath", pubKeyPath+"/c4gh.pub") - pkBytes, err := GetC4GHPublicKey() + pkBytes, err := GetC4GHPublicKey("sync") assert.NoError(ts.T(), err) assert.NotNil(ts.T(), pkBytes) assert.Equal(ts.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") From e7b3a88cfd554b50427ebdd1d4babc0c874d3078 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 15:51:43 +0200 Subject: [PATCH 065/184] get keyhash for a file from db --- sda/internal/database/db_functions.go | 32 +++++++++++++++++++ sda/internal/database/db_functions_test.go | 37 ++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 40372c8d9..0d510fb2e 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -937,6 +937,38 @@ func (dbs *SDAdb) addKeyHash(keyHash, keyDescription string) error { return nil } +// GetKeyHash wraps getKeyHash with exponential stand-off retries +func (dbs *SDAdb) GetKeyHash(fileID string) (string, error) { + var ( + keyHash string + err error + ) + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + keyHash, err = dbs.getKeyHash(fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return keyHash, err +} + +// getKeyHash gets the c4gh key hash corresponding to the fileID in the files table +func (dbs *SDAdb) getKeyHash(fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const query = "SELECT key_hash from sda.files WHERE id = $1;" + var keyHash string + err := db.QueryRow(query, fileID).Scan(&keyHash) + if err != nil { + return "", err + } + + return keyHash, nil +} func (dbs *SDAdb) SetKeyHash(keyHash, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index bbdbc7357..82a3a3caa 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -830,6 +830,43 @@ func (suite *DatabaseTests) TestSetKeyHash_wrongHash() { assert.ErrorContains(suite.T(), err, "violates foreign key constraint") } +func (suite *DatabaseTests) TestGetKeyHash() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + // Register a new key and a new file + keyHex := `6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc509` + keyDescription := "this is a test key" + err = db.addKeyHash(keyHex, keyDescription) + assert.NoError(suite.T(), err, "failed to register key in database") + fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + err = db.SetKeyHash(keyHex, fileID) + + // Test happy path + keyHash, err := db.GetKeyHash(fileID) + assert.NoError(suite.T(), err, "Could not get key hash") + assert.Equal(suite.T(), keyHex, keyHash) + db.Close() +} + +func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + // Register a new key and a new file + keyHex := `6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc509` + keyDescription := "this is a test key" + err = db.addKeyHash(keyHex, keyDescription) + assert.NoError(suite.T(), err, "failed to register key in database") + fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + err = db.SetKeyHash(keyHex, fileID) + + // Test that using an unknown fileID produces an error + _, err = db.GetKeyHash("097e1dc9-6b42-42bf-966d-dece6fefda09") + assert.ErrorContains(suite.T(), err, "no rows in result set") + +} + func (suite *DatabaseTests) TestListDatasets() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) From 9d2496e1ea2b974808e98c29ee3a94cb62619fb8 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 16:04:26 +0200 Subject: [PATCH 066/184] get stableID by FileID from db --- sda/internal/database/db_functions.go | 28 ++++++++++++++++++++ sda/internal/database/db_functions_test.go | 30 ++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0d510fb2e..61d41bf4e 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -63,6 +63,34 @@ func (dbs *SDAdb) getFileID(corrID string) (string, error) { return fileID, nil } +func (dbs *SDAdb) GetFileIDbyAccessionID(accessionID string) (string, error) { + var ( + err error + count int + ID string + ) + + for count == 0 || (err != nil && count < RetryTimes) { + ID, err = dbs.getFileIDbyAccessionID(accessionID) + count++ + } + + return ID, err +} +func (dbs *SDAdb) getFileIDbyAccessionID(accessionID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const getFileID = "SELECT id FROM sda.files where stable_id = $1;" + + var fileID string + err := db.QueryRow(getFileID, accessionID).Scan(&fileID) + if err != nil { + return "", err + } + + return fileID, nil +} + // GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID // and that is not yet archived func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, error) { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 82a3a3caa..5cf8453cf 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -55,6 +55,36 @@ func (suite *DatabaseTests) TestGetFileID() { assert.Equal(suite.T(), fileID, fID) } +func (suite *DatabaseTests) TestGetFileIDbyAccessionID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + stableID := "TEST:000-1234-4567" + err = db.SetAccessionID(stableID, fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + assert.Equal(suite.T(), fileID, retrievedFileID) +} + +func (suite *DatabaseTests) TestGetFileIDbyAccessionID_nonexistentID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + _, err = db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + stableID := "TEST:000-1234-4567" + retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) + assert.ErrorContains(suite.T(), err, "no rows in result set") + assert.Equal(suite.T(), "", retrievedFileID) +} + func (suite *DatabaseTests) TestUpdateFileEventLog() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) From 344fc06a833e922a27b46af03a523a154685e1f0 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 16:18:19 +0200 Subject: [PATCH 067/184] close db connection after each test to avoid too many client connections error --- sda/internal/database/db_functions_test.go | 96 ++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 5cf8453cf..4968e5952 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -37,6 +37,8 @@ func (suite *DatabaseTests) TestRegisterFile() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='registered')", fileID).Scan(&exists) assert.NoError(suite.T(), err, "Failed to check if registered file event exists") assert.True(suite.T(), exists, "RegisterFile() did not insert a row into sda.file_event_log with id: "+fileID) + + db.Close() } func (suite *DatabaseTests) TestGetFileID() { @@ -53,6 +55,8 @@ func (suite *DatabaseTests) TestGetFileID() { fID, err := db.GetFileID(corrID) assert.NoError(suite.T(), err, "GetFileId failed") assert.Equal(suite.T(), fileID, fID) + + db.Close() } func (suite *DatabaseTests) TestGetFileIDbyAccessionID() { @@ -69,6 +73,8 @@ func (suite *DatabaseTests) TestGetFileIDbyAccessionID() { retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), fileID, retrievedFileID) + + db.Close() } func (suite *DatabaseTests) TestGetFileIDbyAccessionID_nonexistentID() { @@ -83,6 +89,8 @@ func (suite *DatabaseTests) TestGetFileIDbyAccessionID_nonexistentID() { retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) assert.ErrorContains(suite.T(), err, "no rows in result set") assert.Equal(suite.T(), "", retrievedFileID) + + db.Close() } func (suite *DatabaseTests) TestUpdateFileEventLog() { @@ -107,6 +115,8 @@ func (suite *DatabaseTests) TestUpdateFileEventLog() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='uploaded')", fileID).Scan(&exists) assert.NoError(suite.T(), err, "Failed to check if uploaded file event exists") assert.True(suite.T(), exists, "UpdateFileEventLog() did not insert a row into sda.file_event_log with id: "+fileID) + + db.Close() } func (suite *DatabaseTests) TestStoreHeader() { @@ -123,6 +133,8 @@ func (suite *DatabaseTests) TestStoreHeader() { // store header for non existing entry err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, "00000000-0000-0000-0000-000000000000") assert.EqualError(suite.T(), err, "something went wrong with the query zero rows were changed") + + db.Close() } func (suite *DatabaseTests) TestSetArchived() { @@ -142,6 +154,8 @@ func (suite *DatabaseTests) TestSetArchived() { err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") + + db.Close() } func (suite *DatabaseTests) TestGetFileStatus() { @@ -159,6 +173,8 @@ func (suite *DatabaseTests) TestGetFileStatus() { status, err := db.GetFileStatus(corrID) assert.NoError(suite.T(), err, "failed to get file status") assert.Equal(suite.T(), "downloaded", status) + + db.Close() } func (suite *DatabaseTests) TestGetHeader() { @@ -175,6 +191,8 @@ func (suite *DatabaseTests) TestGetHeader() { header, err := db.GetHeader(fileID) assert.NoError(suite.T(), err, "failed to get file header") assert.Equal(suite.T(), []byte{15, 45, 20, 40, 48}, header) + + db.Close() } func (suite *DatabaseTests) TestSetVerified() { @@ -191,6 +209,8 @@ func (suite *DatabaseTests) TestSetVerified() { err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + + db.Close() } func (suite *DatabaseTests) TestGetArchived() { @@ -212,6 +232,8 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), 1000, fileSize) assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", filePath) + + db.Close() } func (suite *DatabaseTests) TestSetAccessionID() { @@ -230,6 +252,8 @@ func (suite *DatabaseTests) TestSetAccessionID() { stableID := "TEST:000-1234-4567" err = db.SetAccessionID(stableID, fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + db.Close() } func (suite *DatabaseTests) TestCheckAccessionIDExists() { @@ -256,6 +280,8 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { duplicate, err := db.CheckAccessionIDExists(stableID, uuid.New().String()) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), "duplicate", duplicate) + + db.Close() } func (suite *DatabaseTests) TestGetFileInfo() { @@ -287,6 +313,8 @@ func (suite *DatabaseTests) TestGetFileInfo() { assert.Equal(suite.T(), "/tmp/TestGetFileInfo.c4gh", info.Path) assert.Equal(suite.T(), "11c94bc7fb13afeb2b3fb16c1dbe9206dc09560f1b31420f2d46210ca4ded0a8", info.ArchiveChecksum) assert.Equal(suite.T(), "a671218c2418aa51adf97e33c5c91a720289ba3c9fd0d36f6f4bf9610730749f", info.DecryptedChecksum) + + db.Close() } func (suite *DatabaseTests) TestMapFilesToDataset() { @@ -324,6 +352,8 @@ func (suite *DatabaseTests) TestMapFilesToDataset() { suite.FailNow("failed to get dataset members from database") } assert.Equal(suite.T(), 5, dsMembers) + + db.Close() } func (suite *DatabaseTests) TestGetInboxPath() { @@ -346,6 +376,8 @@ func (suite *DatabaseTests) TestGetInboxPath() { assert.NoError(suite.T(), err, "getInboxPath failed") assert.Contains(suite.T(), path, "/testuser/TestGetInboxPath") } + + db.Close() } func (suite *DatabaseTests) TestUpdateDatasetEvent() { @@ -379,6 +411,8 @@ func (suite *DatabaseTests) TestUpdateDatasetEvent() { err = db.UpdateDatasetEvent(dID, "deprecated", "{\"type\": \"deprecate\"}") assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + db.Close() } func (suite *DatabaseTests) TestGetHeaderForStableID() { @@ -399,6 +433,8 @@ func (suite *DatabaseTests) TestGetHeaderForStableID() { header, err := db.GetHeaderForStableID("TEST:010-1234-4567") assert.NoError(suite.T(), err, "failed to get header for stable ID: %v", err) assert.Equal(suite.T(), header, []byte("HEADER"), "did not get expected header") + + db.Close() } func (suite *DatabaseTests) TestGetSyncData() { @@ -427,6 +463,8 @@ func (suite *DatabaseTests) TestGetSyncData() { assert.Equal(suite.T(), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", fileData.Checksum, "did not get expected file checksum") assert.Equal(suite.T(), "/testuser/TestGetGetSyncData.c4gh", fileData.FilePath, "did not get expected file path") assert.Equal(suite.T(), "testuser", fileData.User, "did not get expected user") + + db.Close() } func (suite *DatabaseTests) TestCheckIfDatasetExists() { @@ -460,6 +498,8 @@ func (suite *DatabaseTests) TestCheckIfDatasetExists() { ok, err = db.checkIfDatasetExists("missing dataset") assert.NoError(suite.T(), err, "check if dataset exists failed") assert.Equal(suite.T(), ok, false) + + db.Close() } func (suite *DatabaseTests) TestGetArchivePath() { @@ -481,6 +521,8 @@ func (suite *DatabaseTests) TestGetArchivePath() { path, err := db.getArchivePath("acession-0001") assert.NoError(suite.T(), err, "getArchivePath failed") assert.Equal(suite.T(), path, corrID) + + db.Close() } func (suite *DatabaseTests) TestGetUserFiles() { @@ -520,6 +562,8 @@ func (suite *DatabaseTests) TestGetUserFiles() { filteredFilelist, err := db.GetUserFiles(testUser, fmt.Sprintf("%s/submission_b", testUser), true) assert.NoError(suite.T(), err, "failed to get file list") assert.Equal(suite.T(), 3, len(filteredFilelist), "file list is of incorrect length") + + db.Close() } func (suite *DatabaseTests) TestGetCorrID() { @@ -537,6 +581,8 @@ func (suite *DatabaseTests) TestGetCorrID() { corrID, err := db.GetCorrID(user, filePath, "") assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") assert.Equal(suite.T(), fileID, corrID) + + db.Close() } func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { @@ -576,6 +622,8 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { corrID, err := db.GetCorrID(user, filePath, "") assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") assert.Equal(suite.T(), fileID2, corrID) + + db.Close() } func (suite *DatabaseTests) TestGetCorrID_wrongFilePath() { @@ -593,6 +641,8 @@ func (suite *DatabaseTests) TestGetCorrID_wrongFilePath() { corrID, err := db.GetCorrID(user, "/testuser/file20.c4gh", "") assert.EqualError(suite.T(), err, "sql: no rows in result set") assert.Equal(suite.T(), "", corrID) + + db.Close() } func (suite *DatabaseTests) TestGetCorrID_fileWithAccessionID() { @@ -614,6 +664,8 @@ func (suite *DatabaseTests) TestGetCorrID_fileWithAccessionID() { corrID, err := db.GetCorrID(user, filePath, "stableID") assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") assert.Equal(suite.T(), fileID, corrID) + + db.Close() } func (suite *DatabaseTests) TestListActiveUsers() { @@ -658,6 +710,8 @@ func (suite *DatabaseTests) TestListActiveUsers() { suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } + + db.Close() } err = db.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"}) @@ -673,6 +727,8 @@ func (suite *DatabaseTests) TestListActiveUsers() { userList, err := db.ListActiveUsers() assert.NoError(suite.T(), err, "failed to list users from DB") assert.Equal(suite.T(), 3, len(userList)) + + db.Close() } func (suite *DatabaseTests) TestGetDatasetStatus() { @@ -745,6 +801,8 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { status, err = db.GetDatasetStatus(dID) assert.NoError(suite.T(), err, "got (%v) when no error weas expected") assert.Equal(suite.T(), "deprecated", status) + + db.Close() } func (suite *DatabaseTests) TestAddKeyHash() { @@ -762,6 +820,8 @@ func (suite *DatabaseTests) TestAddKeyHash() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.encryption_keys WHERE key_hash=$1 AND description=$2)", keyHex, keyDescription).Scan(&exists) assert.NoError(suite.T(), err, "failed to verify key hash existence") assert.True(suite.T(), exists, "key hash was not added to the database") + + db.Close() } func (suite *DatabaseTests) TestListKeyHashes() { @@ -782,6 +842,8 @@ func (suite *DatabaseTests) TestListKeyHashes() { hashList[0].CreatedAt = ct.Format(time.DateOnly) assert.NoError(suite.T(), err, "failed to verify key hash existence") assert.Equal(suite.T(), expectedResponse, hashList[0], "key hash was not added to the database") + + db.Close() } func (suite *DatabaseTests) TestListKeyHashes_emptyTable() { @@ -791,6 +853,8 @@ func (suite *DatabaseTests) TestListKeyHashes_emptyTable() { hashList, err := db.ListKeyHashes() assert.NoError(suite.T(), err, "failed to verify key hash existence") assert.Equal(suite.T(), []C4ghKeyHash{}, hashList, "fuu") + + db.Close() } func (suite *DatabaseTests) TestDeprecateKeyHashes() { @@ -799,6 +863,8 @@ func (suite *DatabaseTests) TestDeprecateKeyHashes() { assert.NoError(suite.T(), db.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc32", "this is a test key"), "failed to register key in database") assert.NoError(suite.T(), db.DeprecateKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc32"), "failure when deprecating keyhash") + + db.Close() } func (suite *DatabaseTests) TestDeprecateKeyHashes_wrongHash() { @@ -807,6 +873,8 @@ func (suite *DatabaseTests) TestDeprecateKeyHashes_wrongHash() { assert.NoError(suite.T(), db.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc11", "this is a another key"), "failed to register key in database") assert.EqualError(suite.T(), db.DeprecateKeyHash("wr0n6h4sh"), "key hash not found or already deprecated", "failure when deprecating non existing keyhash") + + db.Close() } func (suite *DatabaseTests) TestDeprecateKeyHashes_alreadyDeprecated() { @@ -818,6 +886,8 @@ func (suite *DatabaseTests) TestDeprecateKeyHashes_alreadyDeprecated() { // we should not be able to change the deprecation date assert.EqualError(suite.T(), db.DeprecateKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc54"), "key hash not found or already deprecated", "failure when deprecating keyhash") + + db.Close() } func (suite *DatabaseTests) TestSetKeyHash() { @@ -841,6 +911,8 @@ func (suite *DatabaseTests) TestSetKeyHash() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.files WHERE key_hash=$1 AND id=$2)", keyHex, fileID).Scan(&exists) assert.NoError(suite.T(), err, "failed to verify key hash set for file") assert.True(suite.T(), exists, "key hash was not set for file in the database") + + db.Close() } func (suite *DatabaseTests) TestSetKeyHash_wrongHash() { @@ -858,6 +930,8 @@ func (suite *DatabaseTests) TestSetKeyHash_wrongHash() { newKeyHex := "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc502" err = db.SetKeyHash(newKeyHex, fileID) assert.ErrorContains(suite.T(), err, "violates foreign key constraint") + + db.Close() } func (suite *DatabaseTests) TestGetKeyHash() { @@ -876,6 +950,7 @@ func (suite *DatabaseTests) TestGetKeyHash() { keyHash, err := db.GetKeyHash(fileID) assert.NoError(suite.T(), err, "Could not get key hash") assert.Equal(suite.T(), keyHex, keyHash) + db.Close() } @@ -895,6 +970,7 @@ func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { _, err = db.GetKeyHash("097e1dc9-6b42-42bf-966d-dece6fefda09") assert.ErrorContains(suite.T(), err, "no rows in result set") + db.Close() } func (suite *DatabaseTests) TestListDatasets() { @@ -979,6 +1055,8 @@ func (suite *DatabaseTests) TestListDatasets() { assert.NoError(suite.T(), err, "got (%v) when listing datasets", err) assert.Equal(suite.T(), "test-get-dataset-01", datasets[0].DatasetID) assert.Equal(suite.T(), "registered", datasets[1].Status) + + db.Close() } func (suite *DatabaseTests) TestListUserDatasets() { @@ -1071,6 +1149,8 @@ func (suite *DatabaseTests) TestListUserDatasets() { assert.NoError(suite.T(), err, "got (%v) when listing datasets for a user", err) assert.Equal(suite.T(), 2, len(datasets)) assert.Equal(suite.T(), "test-user-dataset-01", datasets[0].DatasetID) + + db.Close() } func (suite *DatabaseTests) TestUpdateUserInfo() { @@ -1091,6 +1171,8 @@ func (suite *DatabaseTests) TestUpdateUserInfo() { err = db.DB.QueryRow("SELECT name FROM sda.userinfo WHERE id=$1", userID).Scan(&name2) assert.NoError(suite.T(), err, "could not select user info: %v", err) assert.Equal(suite.T(), name, name2, "user info table did not update correctly") + + db.Close() } func (suite *DatabaseTests) TestUpdateUserInfo_newInfo() { @@ -1121,6 +1203,8 @@ func (suite *DatabaseTests) TestUpdateUserInfo_newInfo() { err = db.DB.QueryRow("SELECT groups FROM sda.userinfo WHERE id=$1", userID).Scan(pq.Array(&dbgroups)) assert.NoError(suite.T(), err) assert.Equal(suite.T(), groups, dbgroups) + + db.Close() } func (suite *DatabaseTests) TestGetReVerificationData() { @@ -1159,6 +1243,8 @@ func (suite *DatabaseTests) TestGetReVerificationData() { data, err := db.GetReVerificationData(accession) assert.NoError(suite.T(), err, "failed to get verification data") assert.Equal(suite.T(), "/archive/TestGetReVerificationData.c4gh", data.ArchivePath) + + db.Close() } func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { @@ -1198,6 +1284,8 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { data, err := db.GetReVerificationData("accession") assert.EqualError(suite.T(), err, "sql: no rows in result set") assert.Equal(suite.T(), schema.IngestionVerification{}, data) + + db.Close() } func (suite *DatabaseTests) TestGetDecryptedChecksum() { @@ -1233,6 +1321,8 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { checksum, err := db.GetDecryptedChecksum(fileID) assert.NoError(suite.T(), err, "failed to get verification data") assert.Equal(suite.T(), fmt.Sprintf("%x", decSha.Sum(nil)), checksum) + + db.Close() } func (suite *DatabaseTests) TestGetDsatasetFiles() { @@ -1291,6 +1381,8 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { accessions, err := db.GetDatasetFiles(dID) assert.NoError(suite.T(), err, "failed to get accessions for a dataset") assert.Equal(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) + + db.Close() } func (suite *DatabaseTests) TestGetInboxFilePathFromID() { @@ -1315,6 +1407,8 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { assert.NoError(suite.T(), err) _, err = db.getInboxFilePathFromID(user, fileID) assert.Error(suite.T(), err) + + db.Close() } func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { @@ -1351,4 +1445,6 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "archived") assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileID2) + + db.Close() } From b3e0d1b386095dab1eae6d8c3d27fa016db8fc96 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 16:23:02 +0200 Subject: [PATCH 068/184] add rotatekey stream and queue --- rabbitmq/definitions.json | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/rabbitmq/definitions.json b/rabbitmq/definitions.json index 9be22ac25..f19aff92a 100644 --- a/rabbitmq/definitions.json +++ b/rabbitmq/definitions.json @@ -54,6 +54,21 @@ "src-uri": "amqp:///sda" }, "vhost": "sda" + }, + { + "component": "shovel", + "name": "rotatekey", + "value": { + "ack-mode": "on-confirm", + "dest-queue": "rotatekey", + "dest-protocol": "amqp091", + "dest-uri": "amqp:///sda", + "src-delete-after": "never", + "src-protocol": "amqp091", + "src-queue": "rotatekey_stream", + "src-uri": "amqp:///sda" + }, + "vhost": "sda" } ], "global_parameters": [], @@ -149,6 +164,23 @@ "auto_delete": false, "arguments": {} }, + { + "name": "rotatekey", + "vhost": "sda", + "durable": true, + "auto_delete": false, + "arguments": {} + }, + { + "name": "rotatekey_stream", + "vhost": "sda", + "durable": true, + "auto_delete": false, + "arguments": { + "x-max-age": "1M", + "x-queue-type": "stream" + } + }, { "name": "catch_all.dead", "vhost": "sda", @@ -257,6 +289,14 @@ "destination": "verified", "routing_key": "verified" }, + { + "source": "sda", + "vhost": "sda", + "destination_type": "queue", + "arguments": {}, + "destination": "rotatekey_stream", + "routing_key": "rotatekey" + }, { "source": "sda.dead", "vhost": "sda", From 906641ab569e5274fb5f09d6c901b04402994b65 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 17:57:37 +0200 Subject: [PATCH 069/184] create role rotatekey for sda schema --- postgresql/initdb.d/04_grants.sql | 14 ++++++++- postgresql/migratedb.d/18.sql | 51 +++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 postgresql/migratedb.d/18.sql diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index a04fe93f3..6fed60e31 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -126,6 +126,19 @@ GRANT UPDATE ON local_ega.files TO mapper; -------------------------------------------------------------------------------- +CREATE ROLE rotatekey; + +GRANT USAGE ON SCHEMA sda TO rotatekey; +GRANT INSERT ON sda.files TO rotatekey; +GRANT SELECT ON sda.files TO rotatekey; +GRANT UPDATE ON sda.files TO rotatekey; +GRANT SELECT ON sda.checksums TO rotatekey; +GRANT USAGE, SELECT ON SEQUENCE sda.checksums_id_seq TO rotatekey; +GRANT SELECT ON sda.file_event_log TO rotatekey; +GRANT SELECT ON sda.encryption_keys TO rotatekey; + +-------------------------------------------------------------------------------- + CREATE ROLE sync; -- uses: db.GetArchived GRANT USAGE ON SCHEMA sda TO sync; @@ -140,7 +153,6 @@ GRANT UPDATE ON local_ega.main TO sync; -------------------------------------------------------------------------------- - CREATE ROLE download; GRANT USAGE ON SCHEMA sda TO download; diff --git a/postgresql/migratedb.d/18.sql b/postgresql/migratedb.d/18.sql new file mode 100644 index 000000000..92effc43f --- /dev/null +++ b/postgresql/migratedb.d/18.sql @@ -0,0 +1,51 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 17; + changes VARCHAR := 'Create rotatekey role and grant it priviledges to sda tables'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + -- Temporary function for creating roles if they do not already exist. + CREATE FUNCTION create_role_if_not_exists(role_name NAME) RETURNS void AS $created$ + BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = role_name) THEN + RAISE NOTICE 'Role "%" already exists. Skipping.', role_name; + ELSE + BEGIN + EXECUTE format('CREATE ROLE %I', role_name); + EXCEPTION + WHEN duplicate_object THEN + RAISE NOTICE 'Role "%" was just created by a concurrent transaction. Skipping.', role_name; + END; + END IF; + END; + $created$ LANGUAGE plpgsql; + + PERFORM create_role_if_not_exists('rotatekey'); + + GRANT USAGE ON SCHEMA sda TO rotatekey; + GRANT INSERT ON sda.files TO rotatekey; + GRANT SELECT ON sda.files TO rotatekey; + GRANT UPDATE ON sda.files TO rotatekey; + GRANT SELECT ON sda.checksums TO rotatekey; + GRANT USAGE, SELECT ON SEQUENCE sda.checksums_id_seq TO rotatekey; + GRANT SELECT ON sda.file_event_log TO rotatekey; + GRANT SELECT ON sda.encryption_keys TO rotatekey; + + -- Drop temporary user creation function + DROP FUNCTION create_role_if_not_exists; + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ From 108c41bef192b6f64a84f277dc87574f1b7ae11e Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 18:42:27 +0200 Subject: [PATCH 070/184] add rotatekey service --- sda/cmd/rotatekey/rotatekey.go | 243 +++++++++++++++++++++++++++++++++ sda/internal/config/config.go | 25 ++++ 2 files changed, 268 insertions(+) create mode 100644 sda/cmd/rotatekey/rotatekey.go diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go new file mode 100644 index 000000000..57b53ad53 --- /dev/null +++ b/sda/cmd/rotatekey/rotatekey.go @@ -0,0 +1,243 @@ +// The rotatekey service accepts messages for files mapped to a dataset, +// re-encrypts their header with a configured public key and stores it +// in the database together with the key-hash of the rotation key. +// I then sends a message to verify so the file is re-verified. + +package main + +import ( + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/neicnordic/crypt4gh/model/headers" + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/chacha20poly1305" +) + +var ( + err error + publicKey *[32]byte + archiveKeyList []*[32]byte + db *database.SDAdb + conf *config.Config +) + +func main() { + forever := make(chan bool) + conf, err = config.NewConfig("rotatekey") + if err != nil { + log.Fatal(err) + } + mq, err := broker.NewMQ(conf.Broker) + if err != nil { + log.Fatal(err) + } + db, err = database.NewSDAdb(conf.Database) + if err != nil { + log.Fatal(err) + } + + archiveKeyList, err = config.GetC4GHprivateKeys() + if err != nil || len(archiveKeyList) == 0 { + log.Fatal("no C4GH private keys configured") + } + + publicKey, err = config.GetC4GHPublicKey("rotatekey") + if err != nil { + log.Fatal(err) + } + + // Check that the rotation pub key hash exists in the database + keyhash := hex.EncodeToString(publicKey[:]) + hashes, err := db.ListKeyHashes() + if err != nil { + log.Errorln(err.Error()) + } + found := false + for n := range hashes { + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt != "" { + log.Fatal("the crypt4gh rotate key hash has been deprecated") + } + + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt == "" { + found = true + + break + } + } + if !found { + log.Fatal("the crypt4gh rotate key hash is not registered") + } + + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + go func() { + connError := mq.ConnectionWatcher() + log.Error(connError) + forever <- false + }() + + go func() { + connError := mq.ChannelWatcher() + log.Error(connError) + forever <- false + }() + + log.Info("Starting rotatekey service") + var message schema.DatasetMapping + + go func() { + messages, err := mq.GetMessages(conf.Broker.Queue) + if err != nil { + log.Fatal(err) + } + for delivered := range messages { + log.Debugf("Received a message (corr-id: %s, message: %s)", + delivered.CorrelationId, + delivered.Body) + + err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (dataset-mapping) failed, reason: %v", err) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed in rotatekey service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + + for _, aID := range message.AccessionIDs { + + fileID, err := db.GetFileIDbyAccessionID(aID) + if err != nil { + log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) + } + + // Check that the file is not already encrypted with the target key + oldKeyHash, err := db.GetKeyHash(fileID) + if oldKeyHash == keyhash { + log.Errorf("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following already encrypted with key error message") + } + + continue + } + + newHeader, err := reencryptFileHeader(aID) + if err != nil { + log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following reencryptFiles error message") + } + + continue + } + + // Rotate header in database + if err := db.StoreHeader(newHeader, fileID); err != nil { + log.Errorf("StoreHeader failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following storeheader error message") + } + + continue + } + + // Rotate keyhash + if err := db.SetKeyHash(keyhash, fileID); err != nil { + log.Errorf("SetKeyHash failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following setKeyHash error message") + } + + continue + } + + // Send re-verify message + reVerify, err := db.GetReVerificationData(aID) + if err != nil { + log.Errorf("GetReVerificationData failed for file-id: %s, reason: %v", fileID, err) + + continue + } + + reVerifyMsg, _ := json.Marshal(&reVerify) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), reVerifyMsg) + if err != nil { + log.Errorf("Validation of outgoing re-verify message failed, reason: %v", err) + + continue + } + + corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) + if err != nil { + log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) + + continue + } + if err := mq.SendMessage(corrID, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + continue + } + + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + } + }() + + <-forever +} + +func reencryptFileHeader(stableID string) ([]byte, error) { + log.Debugf("rotating c4gh key for file with stable-id: %s", stableID) + + header, err := db.GetHeaderForStableID(stableID) + if err != nil { + return nil, err + } + + // determine decryption key + var key *[32]byte + for _, k := range archiveKeyList { + size, err := headers.EncryptedSegmentSize(header, *k) + if (err == nil) && (size != 0) { + key = k + + break + } + } + + pubkeyList := [][chacha20poly1305.KeySize]byte{*publicKey} + newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) + if err != nil { + return nil, err + } + + return newHeader, nil +} diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index c0a715cb7..1e4bc8658 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -389,6 +389,20 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = []string{ "c4gh.privateKeys", } + case "rotatekey": + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "c4gh.rotatePubKeyPath", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } case "s3inbox": requiredConfVars = []string{ "broker.host", @@ -650,6 +664,17 @@ func NewConfig(app string) (*Config, error) { if err != nil { return nil, err } + case "rotatekey": + if err := c.configBroker(); err != nil { + return nil, err + } + + err := c.configDatabase() + if err != nil { + return nil, err + } + + c.configSchemas() case "s3inbox": err := c.configBroker() if err != nil { From 1ecd711f6d5e37ed773dcda1ec37991c271ec4cd Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 1 Sep 2025 09:03:46 +0200 Subject: [PATCH 071/184] add setup for rotatekey integration tests --- .../scripts/make_db_credentials.sh | 2 +- .../scripts/make_sda_credentials.sh | 7 +++++- .github/integration/sda-s3-integration.yml | 23 +++++++++++++++++++ .github/integration/sda/config.yaml | 3 +++ 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/make_db_credentials.sh b/.github/integration/scripts/make_db_credentials.sh index 4350f1d17..545319ffe 100644 --- a/.github/integration/scripts/make_db_credentials.sh +++ b/.github/integration/scripts/make_db_credentials.sh @@ -4,7 +4,7 @@ set -e apt-get -o DPkg::Lock::Timeout=60 update > /dev/null apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client >/dev/null -for n in api auth download finalize inbox ingest mapper sync verify; do +for n in api auth download finalize inbox ingest mapper rotatekey sync verify; do echo "creating credentials for: $n" psql -U postgres -h migrate -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 9beaec98d..923b35c1c 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -14,7 +14,7 @@ apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl post pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc requests > /dev/null -for n in api auth download finalize inbox ingest mapper sync verify; do +for n in api auth download finalize inbox ingest mapper rotatekey sync verify; do echo "creating credentials for: $n" psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" @@ -121,6 +121,11 @@ if [ ! -f "/shared/sync.sec.pem" ]; then /shared/crypt4gh generate -n /shared/sync -p syncPass fi +if [ ! -f "/shared/rotatekey.sec.pem" ]; then + echo "creating rotatekey crypth4gh key" + /shared/crypt4gh generate -n /shared/rotatekey -p rotatekeyPass +fi + if [ ! -f "/shared/keys/ssh" ]; then ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N "" pubKey="$(cat /shared/keys/ssh.pub)" diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 1d2e7a5cf..e35cbbeb7 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -253,6 +253,29 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + rotatekey: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [sda-rotatekey] + container_name: rotatekey + depends_on: + credentials: + condition: service_completed_successfully + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + environment: + - BROKER_PASSWORD=rotatekey + - BROKER_USER=rotatekey + - BROKER_QUEUE=rotatekey + - BROKER_ROUTINGKEY=rotatekey + - DB_PASSWORD=rotatekey + - DB_USER=rotatekey + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + cega-nss: container_name: cega-nss depends_on: diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index e5473a023..066c4d11b 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -74,11 +74,14 @@ c4gh: filePath: /shared/c4gh.sec.pem passphrase: "c4ghpass" syncPubKeyPath: /shared/sync.pub.pem + rotatePubKeyPath: /shared/rotatekey.pub.pem privateKeys: - filePath: /shared/c4gh.sec.pem passphrase: "c4ghpass" - filePath: /shared/c4gh1.sec.pem passphrase: "c4ghpass" + - filePath: /shared/rotatekey.sec.pem + passphrase: "rotatekeyPass" oidc: id: XC56EL11xx From 659f12ff3afaa85c785a9f43d7a5d6645b5e9c41 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 23 Sep 2025 12:27:50 +0200 Subject: [PATCH 072/184] cleanup temporary create_role function so that the migration flow works when later adding a new role --- postgresql/migratedb.d/14.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql/migratedb.d/14.sql b/postgresql/migratedb.d/14.sql index f39a19af1..d2d2204e9 100644 --- a/postgresql/migratedb.d/14.sql +++ b/postgresql/migratedb.d/14.sql @@ -41,6 +41,9 @@ BEGIN GRANT SELECT, INSERT, UPDATE ON sda.userinfo TO auth; GRANT base TO auth; + + -- Drop temporary user creation function + DROP FUNCTION create_role_if_not_exists; ELSE RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; END IF; From 5284c271c19244c39176160f716914f76fdecf81 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 1 Sep 2025 10:56:12 +0200 Subject: [PATCH 073/184] register rotation key in the db during test setup - only register key if it is missing --- .github/integration/scripts/make_sda_credentials.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 923b35c1c..b56dbe16a 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -9,7 +9,7 @@ if [ -n "$PGSSLCERT" ]; then fi apt-get -o DPkg::Lock::Timeout=60 update > /dev/null -apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl postgresql-client >/dev/null +apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl postgresql-client xxd >/dev/null pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc requests > /dev/null @@ -126,6 +126,17 @@ if [ ! -f "/shared/rotatekey.sec.pem" ]; then /shared/crypt4gh generate -n /shared/rotatekey -p rotatekeyPass fi +# register the rotation key in the db +resp=$(psql -U postgres -h postgres -d sda -At -c "SELECT description FROM sda.encryption_keys;") +if ! echo "$resp" | grep -q 'this is the new key to rotate to'; then + rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) + resp=$(psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$rotateKeyHash', 'this is the new key to rotate to');") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert keyhash failed" + exit 1 + fi +fi + if [ ! -f "/shared/keys/ssh" ]; then ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N "" pubKey="$(cat /shared/keys/ssh.pub)" From 194e4502c8d40237ace8b79215ef0cac3440eba1 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 1 Sep 2025 11:09:47 +0200 Subject: [PATCH 074/184] send msg to erorr queue of get keyhash fails --- sda/cmd/rotatekey/rotatekey.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 57b53ad53..2276d57b8 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -128,14 +128,34 @@ func main() { _ = json.Unmarshal(delivered.Body, &message) for _, aID := range message.AccessionIDs { - fileID, err := db.GetFileIDbyAccessionID(aID) if err != nil { log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) } - // Check that the file is not already encrypted with the target key + // Get current keyhash for the file, send to error queue if this fails oldKeyHash, err := db.GetKeyHash(fileID) + if err != nil { + log.Errorf("failed to get keyhash for file with accession-id: %s, reason: %v", aID, err) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to get keyhash in rotatekey service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + + continue + } + + // Check that the file is not already encrypted with the target key if oldKeyHash == keyhash { log.Errorf("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) if err := delivered.Nack(false, false); err != nil { @@ -202,7 +222,6 @@ func main() { continue } - } if err := delivered.Ack(false); err != nil { From 2b5cdc9b449c4939bb4adeb8d366322662cdb7c7 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 1 Sep 2025 14:36:31 +0200 Subject: [PATCH 075/184] fix linting --- sda/internal/config/config.go | 1 - sda/internal/database/db_functions_test.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 1e4bc8658..e15b56266 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -1224,7 +1224,6 @@ func GetC4GHprivateKeys() ([]*[32]byte, error) { // GetC4GHPublicKey reads the c4gh public key func GetC4GHPublicKey(app string) (*[32]byte, error) { - var keyPath string switch app { case "sync": diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 4968e5952..97cc675fe 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -945,6 +945,7 @@ func (suite *DatabaseTests) TestGetKeyHash() { fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) + assert.NoError(suite.T(), err, "failed to set key hash in database") // Test happy path keyHash, err := db.GetKeyHash(fileID) @@ -965,6 +966,7 @@ func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) + assert.NoError(suite.T(), err, "failed to set key hash in database") // Test that using an unknown fileID produces an error _, err = db.GetKeyHash("097e1dc9-6b42-42bf-966d-dece6fefda09") From d8aa86179edda35e1c9eb75f5dca71691edfc8de Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 4 Sep 2025 15:54:23 +0200 Subject: [PATCH 076/184] checks target key hash both at startup and before processing messages --- sda/cmd/rotatekey/rotatekey.go | 75 +++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 2276d57b8..8088d2117 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -8,6 +8,7 @@ package main import ( "encoding/hex" "encoding/json" + "errors" "fmt" "github.com/neicnordic/crypt4gh/model/headers" @@ -52,26 +53,10 @@ func main() { log.Fatal(err) } - // Check that the rotation pub key hash exists in the database - keyhash := hex.EncodeToString(publicKey[:]) - hashes, err := db.ListKeyHashes() + // Check that key is registered in the db at startup + keyhash, err := getKeyHash() if err != nil { - log.Errorln(err.Error()) - } - found := false - for n := range hashes { - if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt != "" { - log.Fatal("the crypt4gh rotate key hash has been deprecated") - } - - if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt == "" { - found = true - - break - } - } - if !found { - log.Fatal("the crypt4gh rotate key hash is not registered") + log.Fatalf("database lookup of the rotation key failed, reason: %v", err) } defer mq.Channel.Close() @@ -124,6 +109,29 @@ func main() { continue } + // Fetch rotate key hash before starting work so that we make sure the hash state + // has not changed since the application startup. + keyhash, err = getKeyHash() + if err != nil { + log.Errorf("database lookup of the rotation key failed, reason: %v", err) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Lookup of rotation key hash failed in rotatekey service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + + continue + } + // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) @@ -131,6 +139,8 @@ func main() { fileID, err := db.GetFileIDbyAccessionID(aID) if err != nil { log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) + + continue } // Get current keyhash for the file, send to error queue if this fails @@ -217,6 +227,7 @@ func main() { continue } + if err := mq.SendMessage(corrID, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) @@ -260,3 +271,29 @@ func reencryptFileHeader(stableID string) ([]byte, error) { return newHeader, nil } + +// Check that the key hash exists in the database +func getKeyHash() (string, error) { + keyhash := hex.EncodeToString(publicKey[:]) + hashes, err := db.ListKeyHashes() + if err != nil { + return "", err + } + found := false + for n := range hashes { + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt != "" { + return "", errors.New("the c4gh key hash has been deprecated") + } + + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt == "" { + found = true + + break + } + } + if !found { + return "", errors.New("the c4gh key hash is not registered") + } + + return keyhash, nil +} From da2978721d71fbf12762526eebb395e0b9033280 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 4 Sep 2025 16:35:11 +0200 Subject: [PATCH 077/184] nack after some error messages --- sda/cmd/rotatekey/rotatekey.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 8088d2117..26a7a33cb 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -139,6 +139,9 @@ func main() { fileID, err := db.GetFileIDbyAccessionID(aID) if err != nil { log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following failed to get fileID from accessionID error message") + } continue } @@ -149,7 +152,7 @@ func main() { log.Errorf("failed to get keyhash for file with accession-id: %s, reason: %v", aID, err) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Failed to get keyhash in rotatekey service", + Error: "Failed to get source key hash in rotatekey service", Reason: err.Error(), OriginalMessage: string(delivered.Body), } @@ -209,6 +212,9 @@ func main() { reVerify, err := db.GetReVerificationData(aID) if err != nil { log.Errorf("GetReVerificationData failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following GetReVerificationData error message") + } continue } @@ -217,6 +223,9 @@ func main() { err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), reVerifyMsg) if err != nil { log.Errorf("Validation of outgoing re-verify message failed, reason: %v", err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after verify schema validation error message") + } continue } @@ -224,12 +233,18 @@ func main() { corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) if err != nil { log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after GetCorrID error message") + } continue } if err := mq.SendMessage(corrID, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after SendMessage error message") + } continue } From c920bd0e9eef901c796c38eaa0b2084295a32509 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 4 Sep 2025 18:52:48 +0200 Subject: [PATCH 078/184] add rotatekey integration tests --- .github/integration/sda-s3-integration.yml | 2 + .github/integration/tests/sda/45_sync_test.sh | 2 +- .../tests/sda/70_rotate_key_test.sh | 171 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 .github/integration/tests/sda/70_rotate_key_test.sh diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index e35cbbeb7..3567b9600 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -407,6 +407,8 @@ services: condition: service_started reencrypt: condition: service_started + rotatekey: + condition: service_started extra_hosts: - "localhost:host-gateway" environment: diff --git a/.github/integration/tests/sda/45_sync_test.sh b/.github/integration/tests/sda/45_sync_test.sh index 4d44aef99..276fbfd3c 100644 --- a/.github/integration/tests/sda/45_sync_test.sh +++ b/.github/integration/tests/sda/45_sync_test.sh @@ -11,7 +11,7 @@ fi # check bucket for synced files for file in NA12878.bai NA12878_20k_b37.bai; do RETRY_TIMES=0 - until [ "$(s3cmd -c direct ls s3://sync/test_dummy.org/"$file")" != "" ]; do + until [ "$(s3cmd -c direct ls s3://sync/"$file")" != "" ]; do RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then echo "::error::Time out while waiting for files to be synced" diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh new file mode 100644 index 000000000..f5153ff74 --- /dev/null +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -0,0 +1,171 @@ +#!/bin/sh +set -e + +if [ -n "$SYNCTEST" ]; then + exit 0 +fi + +cd shared || true + +checkStatus () { + RETRY_TIMES=0 + until [ "$(curl -s -k -H "Authorization: Bearer $token" -X GET http://api:8080/users/test@dummy.org/files | jq | grep -c "$1")" -eq "$2" ]; do + echo "waiting for files to become $1" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to become $1" + exit 1 + fi + sleep 2 + done +} + +# cleanup queues and database +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +for q in accession archived backup completed inbox ingest mappings verified rotatekey; do + curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents" +done +psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files, sda.encryption_keys CASCADE;" + +# register archive and rotation c4gh public keys +token="$(cat /shared/token)" +for keyName in c4gh rotatekey; do + payload=$( + jq -c -n \ + --arg description "this is the $keyName key" \ + --arg pubkey "$( base64 -w0 /shared/"$keyName".pub.pem)" \ + '$ARGS.named' + ) + resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")" + if [ "$resp" != "200" ]; then + echo "Error when adding the $keyName public key hash, expected 200 got: $resp" + exit 1 + fi +done + +# generate and upload files +for file in testfile1 testfile2; do + if [ ! -f "$file" ]; then + dd if=/dev/urandom of="$file" count=10 bs=1M + fi + if [ ! -f "$file.c4gh" ]; then + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" + fi + s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/dataset_rotatekey/ +done +response="$(curl -s -k -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq | grep -c dataset_rotatekey)" +if [ "$response" -ne 2 ]; then + echo "files for rotatekey test failed to upload" + exit 1 +fi + +## ingest and map files to dataset +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/ingest +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "dataset_rotatekey/testfile2.c4gh", "user": "test@dummy.org"}' http://api:8080/file/ingest +checkStatus verified 2 + +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "ROTATE-KEY-01", "filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/accession +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "ROTATE-KEY-02", "filepath": "dataset_rotatekey/testfile2.c4gh", "user": "test@dummy.org"}' http://api:8080/file/accession +checkStatus ready 2 + +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["ROTATE-KEY-01", "ROTATE-KEY-02"], "dataset_id": "KEY-ROTATION-TEST-0001", "user": "test@dummy.org"}' http://api:8080/dataset/create +checkStatus ready 0 + +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +## trigger key rotation +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + +mappings=$( + jq -c -n \ + '$ARGS.positional' \ + --args "ROTATE-KEY-01" \ + --args "ROTATE-KEY-02" +) + +mapping_payload=$( + jq -r -c -n \ + --arg type mapping \ + --arg dataset_id KEY-ROTATION-TEST-0001 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64' +) + +mapping_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "rotatekey" \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" | jq + +# check DB for updated key hash in sda.files +rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") +if "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -neq 2; +then + echo "failed to update the key hash of files" + exit 1 +fi + +# check that files were re-verified +echo "waiting for re-verify to complete" +RETRY_TIMES=0 +until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/archived/ | jq -r '.messages_ready')" -eq 0 ]; do + echo "waiting for re-verify to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for verify to complete" + exit 1 + fi + sleep 2 +done + +# check that no other erros occured +if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')" -ne "$errorStreamSize" ]; then + echo "something went wrong with the key rotation" + exit 1 +fi + +# download files with rotated key, concatenate header and archive body, decrypt and chack + +# get rotated header +psql -U postgres -h postgres -d sda -At -c "select header from sda.files where stable_id='ROTATE-KEY-01';" | xxd -r -p > testfile1_rotated.c4gh +# get archive file +archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where stable_id='ROTATE-KEY-01';") +s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force +# concatenate and decrypt +cat testfile1_rotated.c4gh "$archivePath" > tmp_file && mv tmp_file testfile1_rotated.c4gh +C4GH_PASSPHRASE=rotatekeyPass ./crypt4gh decrypt -f testfile1_rotated.c4gh -s rotatekey.sec.pem + +# check that decrypted file matches the original +if [ ! -f "testfile1_rotated" ]; then + echo "decrypted file testfile1_rotated not found" + exit 1 +fi +if ! cmp -s "testfile1_rotated" "testfile1" ; then + echo "downloaded file is different from the original one" + exit 1 +fi +# compare hashes as well +if [ "$(sha256sum testfile1 | cut -d ' ' -f 1)" != "$(sha256sum testfile1 | cut -d ' ' -f 1)" ]; then + echo "downloaded file has different sha256 hash from the original one" + exit 1 +fi + +echo "Rotate key integration tests completed successfully" From 227e1f2d3edbe2f25797c3f44f7b3dae1bb4f070 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 9 Sep 2025 15:24:39 +0200 Subject: [PATCH 079/184] reencrypt headers using the reencrypt service --- sda/cmd/rotatekey/rotatekey.go | 75 +++++++++++++++++++++++----------- sda/internal/config/config.go | 11 +++++ 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 26a7a33cb..526196cc8 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -6,26 +6,31 @@ package main import ( + "bytes" + "context" + "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" + "time" - "github.com/neicnordic/crypt4gh/model/headers" + "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" log "github.com/sirupsen/logrus" - "golang.org/x/crypto/chacha20poly1305" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) var ( - err error - publicKey *[32]byte - archiveKeyList []*[32]byte - db *database.SDAdb - conf *config.Config + err error + publicKey *[32]byte + db *database.SDAdb + conf *config.Config ) func main() { @@ -43,11 +48,6 @@ func main() { log.Fatal(err) } - archiveKeyList, err = config.GetC4GHprivateKeys() - if err != nil || len(archiveKeyList) == 0 { - log.Fatal("no C4GH private keys configured") - } - publicKey, err = config.GetC4GHPublicKey("rotatekey") if err != nil { log.Fatal(err) @@ -267,19 +267,14 @@ func reencryptFileHeader(stableID string) ([]byte, error) { return nil, err } - // determine decryption key - var key *[32]byte - for _, k := range archiveKeyList { - size, err := headers.EncryptedSegmentSize(header, *k) - if (err == nil) && (size != 0) { - key = k - - break - } + // encode pubkey as pem and then as base64 string + tmp := &bytes.Buffer{} + if err = keys.WriteCrypt4GHX25519PublicKey(tmp, *publicKey); err != nil { + return nil, err } + pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) - pubkeyList := [][chacha20poly1305.KeySize]byte{*publicKey} - newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) + newHeader, err := reencryptHeader(header, pubKeyEncoded) if err != nil { return nil, err } @@ -312,3 +307,37 @@ func getKeyHash() (string, error) { return keyhash, nil } + +// reencryptHeader re-encrypts the header of a file using the public key +// provided in the request header and returns the new header. The function uses +// gRPC to communicate with the re-encrypt service and handles TLS configuration +// if needed. The function also handles the case where the CA certificate is +// provided for secure communication. +func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { + var opts []grpc.DialOption + switch { + case conf.RotateKey.Grpc.ClientCreds != nil: + opts = append(opts, grpc.WithTransportCredentials(conf.RotateKey.Grpc.ClientCreds)) + default: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", conf.RotateKey.Grpc.Host, conf.RotateKey.Grpc.Port), opts...) + if err != nil { + log.Errorf("failed to connect to the reencrypt service, reason: %s", err) + + return nil, err + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.RotateKey.Grpc.Timeout)*time.Second) + defer cancel() + + c := reencrypt.NewReencryptClient(conn) + res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) + if err != nil { + return nil, err + } + + return res.Header, nil +} diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index e15b56266..147030c03 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -50,6 +50,7 @@ type Config struct { SyncAPI SyncAPIConf ReEncrypt ReEncConfig Auth AuthConf + RotateKey RotateKeyConf } type Grpc struct { @@ -70,6 +71,10 @@ type ReEncConfig struct { Timeout int } +type RotateKeyConf struct { + Grpc Grpc +} + type Sync struct { CenterPrefix string Destination storage.Conf @@ -402,6 +407,7 @@ func NewConfig(app string) (*Config, error) { "db.user", "db.password", "db.database", + "grpc.host", } case "s3inbox": requiredConfVars = []string{ @@ -675,6 +681,11 @@ func NewConfig(app string) (*Config, error) { } c.configSchemas() + + c.RotateKey.Grpc, err = configReEncryptClient() + if err != nil { + return nil, err + } case "s3inbox": err := c.configBroker() if err != nil { From b3707c6d99ec79a005034acdcbab0a529f6b9a31 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 9 Sep 2025 15:29:49 +0200 Subject: [PATCH 080/184] add config unittests for rotatekey --- sda/internal/config/config_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index f58b4139d..a724fd249 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -326,6 +326,35 @@ func (ts *ConfigTestSuite) TestSyncConfig() { assert.NotNil(ts.T(), config.Sync.Destination.Posix) assert.Equal(ts.T(), "test", config.Sync.Destination.Posix.Location) } + +func (ts *ConfigTestSuite) TestRotateKeyConfig() { + ts.SetupTest() + // At this point we should fail because we lack configuration + config, err := NewConfig("rotatekey") + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) + + viper.Set("c4gh.rotatePubKeyPath", "/keys/recipient") + config, err = NewConfig("rotatekey") + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Broker) + assert.Equal(ts.T(), "testhost", config.Broker.Host) + assert.Equal(ts.T(), 123, config.Broker.Port) + assert.Equal(ts.T(), "testuser", config.Broker.User) + assert.Equal(ts.T(), "testpassword", config.Broker.Password) + assert.Equal(ts.T(), "routingtest", config.Broker.RoutingKey) + assert.NotNil(ts.T(), config.Database) + assert.Equal(ts.T(), "test", config.Database.Host) + assert.Equal(ts.T(), 123, config.Database.Port) + assert.Equal(ts.T(), "test", config.Database.User) + assert.Equal(ts.T(), "test", config.Database.Password) + assert.Equal(ts.T(), "test", config.Database.Database) + assert.NotNil(ts.T(), config.RotateKey) + assert.NotNil(ts.T(), config.RotateKey.Grpc) + assert.Equal(ts.T(), "reencrypt", config.RotateKey.Grpc.Host) +} + func (ts *ConfigTestSuite) TestGetC4GHPublicKey() { pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" pubKeyPath, _ := os.MkdirTemp("", "pubkey") From f12c28c204d4fafe663d5b7dcb931caf1cce62cc Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 9 Sep 2025 16:11:49 +0200 Subject: [PATCH 081/184] throw an error if call to reencrypt returns an empty header --- sda/cmd/rotatekey/rotatekey.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 526196cc8..7d09f6b8f 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -187,6 +187,15 @@ func main() { continue } + if newHeader == nil { + err = errors.New("reencrypt returned empty header") + log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following reencryptFiles error message") + } + + continue + } // Rotate header in database if err := db.StoreHeader(newHeader, fileID); err != nil { From 1c57182bf94baeba18a7be9c3b97be3142c4cc88 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 12 Sep 2025 15:56:32 +0200 Subject: [PATCH 082/184] enforce processing only one file per message so that messages can be nacked properly upon error --- sda/cmd/rotatekey/rotatekey.go | 223 +++++++++++++++++---------------- 1 file changed, 117 insertions(+), 106 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 7d09f6b8f..133a8ca64 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -30,20 +30,20 @@ var ( err error publicKey *[32]byte db *database.SDAdb - conf *config.Config + Conf *config.Config ) func main() { forever := make(chan bool) - conf, err = config.NewConfig("rotatekey") + Conf, err = config.NewConfig("rotatekey") if err != nil { log.Fatal(err) } - mq, err := broker.NewMQ(conf.Broker) + mq, err := broker.NewMQ(Conf.Broker) if err != nil { log.Fatal(err) } - db, err = database.NewSDAdb(conf.Database) + db, err = database.NewSDAdb(Conf.Database) if err != nil { log.Fatal(err) } @@ -79,7 +79,7 @@ func main() { var message schema.DatasetMapping go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) + messages, err := mq.GetMessages(Conf.Broker.Queue) if err != nil { log.Fatal(err) } @@ -88,7 +88,7 @@ func main() { delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (dataset-mapping) failed, reason: %v", err) // Send the message to an error queue so it can be analyzed. @@ -99,7 +99,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { @@ -122,7 +122,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { @@ -135,128 +135,139 @@ func main() { // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - for _, aID := range message.AccessionIDs { - fileID, err := db.GetFileIDbyAccessionID(aID) - if err != nil { - log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following failed to get fileID from accessionID error message") - } + // We expect only one aID per message so that we handle errors and nacks properly. + // A different json schema seems like a cleaner solution going forward. + if len(message.AccessionIDs) > 1 { + log.Errorf("failed to process message, reason: multiple accession_id's per message is not supported") + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + + continue + } + + aID := message.AccessionIDs[0] - continue + fileID, err := db.GetFileIDbyAccessionID(aID) + if err != nil { + log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following failed to get fileID from accessionID error message") } - // Get current keyhash for the file, send to error queue if this fails - oldKeyHash, err := db.GetKeyHash(fileID) - if err != nil { - log.Errorf("failed to get keyhash for file with accession-id: %s, reason: %v", aID, err) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to get source key hash in rotatekey service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue + continue + } + + // Get current keyhash for the file, send to error queue if this fails + oldKeyHash, err := db.GetKeyHash(fileID) + if err != nil { + log.Errorf("failed to get keyhash for file with accession-id: %s, reason: %v", aID, err) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to get source key hash in rotatekey service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - // Check that the file is not already encrypted with the target key - if oldKeyHash == keyhash { - log.Errorf("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following already encrypted with key error message") - } + continue + } - continue + // Check that the file is not already encrypted with the target key + if oldKeyHash == keyhash { + log.Errorf("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following already encrypted with key error message") } - newHeader, err := reencryptFileHeader(aID) - if err != nil { - log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following reencryptFiles error message") - } + continue + } - continue + newHeader, err := reencryptFile(aID) + if err != nil { + log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following reencryptFiles error message") } - if newHeader == nil { - err = errors.New("reencrypt returned empty header") - log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following reencryptFiles error message") - } - - continue + + continue + } + if newHeader == nil { + err = errors.New("reencrypt returned empty header") + log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following reencryptFiles error message") } - // Rotate header in database - if err := db.StoreHeader(newHeader, fileID); err != nil { - log.Errorf("StoreHeader failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following storeheader error message") - } + continue + } - continue + // Rotate header in database + if err := db.StoreHeader(newHeader, fileID); err != nil { + log.Errorf("StoreHeader failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following storeheader error message") } - // Rotate keyhash - if err := db.SetKeyHash(keyhash, fileID); err != nil { - log.Errorf("SetKeyHash failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following setKeyHash error message") - } + continue + } - continue + // Rotate keyhash + if err := db.SetKeyHash(keyhash, fileID); err != nil { + log.Errorf("SetKeyHash failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following setKeyHash error message") } - // Send re-verify message - reVerify, err := db.GetReVerificationData(aID) - if err != nil { - log.Errorf("GetReVerificationData failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following GetReVerificationData error message") - } + continue + } - continue + // Send re-verify message + reVerify, err := db.GetReVerificationData(aID) + if err != nil { + log.Errorf("GetReVerificationData failed for file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following GetReVerificationData error message") } - reVerifyMsg, _ := json.Marshal(&reVerify) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), reVerifyMsg) - if err != nil { - log.Errorf("Validation of outgoing re-verify message failed, reason: %v", err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after verify schema validation error message") - } + continue + } - continue + reVerifyMsg, _ := json.Marshal(&reVerify) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", Conf.Broker.SchemasPath), reVerifyMsg) + if err != nil { + log.Errorf("Validation of outgoing re-verify message failed, reason: %v", err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after verify schema validation error message") } - corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) - if err != nil { - log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after GetCorrID error message") - } + continue + } - continue + corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) + if err != nil { + log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after GetCorrID error message") } - if err := mq.SendMessage(corrID, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after SendMessage error message") - } + continue + } - continue + if err := mq.SendMessage(corrID, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack after SendMessage error message") } + + continue } if err := delivered.Ack(false); err != nil { @@ -268,7 +279,7 @@ func main() { <-forever } -func reencryptFileHeader(stableID string) ([]byte, error) { +func reencryptFile(stableID string) ([]byte, error) { log.Debugf("rotating c4gh key for file with stable-id: %s", stableID) header, err := db.GetHeaderForStableID(stableID) @@ -325,13 +336,13 @@ func getKeyHash() (string, error) { func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { var opts []grpc.DialOption switch { - case conf.RotateKey.Grpc.ClientCreds != nil: - opts = append(opts, grpc.WithTransportCredentials(conf.RotateKey.Grpc.ClientCreds)) + case Conf.RotateKey.Grpc.ClientCreds != nil: + opts = append(opts, grpc.WithTransportCredentials(Conf.RotateKey.Grpc.ClientCreds)) default: opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", conf.RotateKey.Grpc.Host, conf.RotateKey.Grpc.Port), opts...) + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", Conf.RotateKey.Grpc.Host, Conf.RotateKey.Grpc.Port), opts...) if err != nil { log.Errorf("failed to connect to the reencrypt service, reason: %s", err) @@ -339,7 +350,7 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { } defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.RotateKey.Grpc.Timeout)*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Conf.RotateKey.Grpc.Timeout)*time.Second) defer cancel() c := reencrypt.NewReencryptClient(conn) From a4db996f2538b13be569830a1fa68c5f900525b6 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 15 Sep 2025 00:04:55 +0200 Subject: [PATCH 083/184] rework nacking mechanism - refactor mechanism into a function - always send info-error message upon error --- sda/cmd/rotatekey/rotatekey.go | 139 ++++++++++++--------------------- 1 file changed, 49 insertions(+), 90 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 133a8ca64..e87a40671 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -21,6 +21,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -90,21 +91,8 @@ func main() { err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, reason: %v", err) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed in rotatekey service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + msg := "validation of incoming message (dataset-mapping) failed" + logAndNack(mq, delivered, msg, err) continue } @@ -113,21 +101,8 @@ func main() { // has not changed since the application startup. keyhash, err = getKeyHash() if err != nil { - log.Errorf("database lookup of the rotation key failed, reason: %v", err) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Lookup of rotation key hash failed in rotatekey service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + msg := "database lookup of the rotation key failed" + logAndNack(mq, delivered, msg, err) continue } @@ -138,10 +113,9 @@ func main() { // We expect only one aID per message so that we handle errors and nacks properly. // A different json schema seems like a cleaner solution going forward. if len(message.AccessionIDs) > 1 { - log.Errorf("failed to process message, reason: multiple accession_id's per message is not supported") - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + msg := "failed to process message" + err = errors.New("multiple accession_ids per message is not supported") + logAndNack(mq, delivered, msg, err) continue } @@ -150,10 +124,8 @@ func main() { fileID, err := db.GetFileIDbyAccessionID(aID) if err != nil { - log.Errorf("failed to get file-id for file with accession-id: %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following failed to get fileID from accessionID error message") - } + msg := fmt.Sprintf("failed to get file-id for file with accession-id: %s", aID) + logAndNack(mq, delivered, msg, err) continue } @@ -161,70 +133,47 @@ func main() { // Get current keyhash for the file, send to error queue if this fails oldKeyHash, err := db.GetKeyHash(fileID) if err != nil { - log.Errorf("failed to get keyhash for file with accession-id: %s, reason: %v", aID, err) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to get source key hash in rotatekey service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + msg := fmt.Sprintf("failed to get keyhash for file with accession-id: %s", aID) + logAndNack(mq, delivered, msg, err) continue } // Check that the file is not already encrypted with the target key if oldKeyHash == keyhash { - log.Errorf("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following already encrypted with key error message") - } + msg := fmt.Sprintf("failed to reencrypt file with file-id: %s", fileID) + err = errors.New("already encrypted with the given rotation c4gh key") + logAndNack(mq, delivered, msg, err) continue } newHeader, err := reencryptFile(aID) if err != nil { - log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following reencryptFiles error message") - } + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) + logAndNack(mq, delivered, msg, err) continue } if newHeader == nil { - err = errors.New("reencrypt returned empty header") - log.Errorf("failed to rotate c4gh key for file %s, reason: %v", aID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following reencryptFiles error message") - } + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) + logAndNack(mq, delivered, msg, err) continue } // Rotate header in database if err := db.StoreHeader(newHeader, fileID); err != nil { - log.Errorf("StoreHeader failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following storeheader error message") - } + msg := fmt.Sprintf("StoreHeader failed for file-id: %s", fileID) + logAndNack(mq, delivered, msg, err) continue } // Rotate keyhash if err := db.SetKeyHash(keyhash, fileID); err != nil { - log.Errorf("SetKeyHash failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following setKeyHash error message") - } + msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) + logAndNack(mq, delivered, msg, err) continue } @@ -232,10 +181,8 @@ func main() { // Send re-verify message reVerify, err := db.GetReVerificationData(aID) if err != nil { - log.Errorf("GetReVerificationData failed for file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following GetReVerificationData error message") - } + msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) + logAndNack(mq, delivered, msg, err) continue } @@ -243,29 +190,23 @@ func main() { reVerifyMsg, _ := json.Marshal(&reVerify) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", Conf.Broker.SchemasPath), reVerifyMsg) if err != nil { - log.Errorf("Validation of outgoing re-verify message failed, reason: %v", err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after verify schema validation error message") - } + msg := "Validation of outgoing re-verify message failed" + logAndNack(mq, delivered, msg, err) continue } corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) if err != nil { - log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after GetCorrID error message") - } + msg := fmt.Sprintf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) + logAndNack(mq, delivered, msg, err) continue } if err := mq.SendMessage(corrID, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack after SendMessage error message") - } + msg := "failed to publish message" + logAndNack(mq, delivered, msg, err) continue } @@ -361,3 +302,21 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { return res.Header, nil } + +// Send the message to an error queue so it can be analyzed and then nack message. +func logAndNack(mq *broker.AMQPBroker, delivered amqp091.Delivery, msg string, err error) { + log.Errorf("%s, reason: %v", msg, err) + infoErrorMessage := broker.InfoError{ + Error: msg, + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + body, _ := json.Marshal(infoErrorMessage) + + if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } +} From 556feffd4818d099f0476ea0ff58c0c87477b9f6 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 15 Sep 2025 00:10:16 +0200 Subject: [PATCH 084/184] add error cases in rotatekey integration tests --- .../tests/sda/70_rotate_key_test.sh | 112 ++++++++++++++---- 1 file changed, 90 insertions(+), 22 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index f5153ff74..e664d1082 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -20,6 +20,19 @@ checkStatus () { done } +checkErrors() { + RETRY_TIMES=0 + until [ $(("$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')"-"$errorStreamSize")) -eq 1 ]; do + echo "checking for $1 error" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 20 ]; then + echo "::error::Time out while waiting for error message" + exit 1 + fi + sleep 2 + done +} + # cleanup queues and database URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then @@ -46,32 +59,30 @@ for keyName in c4gh rotatekey; do fi done -# generate and upload files -for file in testfile1 testfile2; do - if [ ! -f "$file" ]; then - dd if=/dev/urandom of="$file" count=10 bs=1M - fi - if [ ! -f "$file.c4gh" ]; then - yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" - fi - s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/dataset_rotatekey/ -done +# generate and upload file +file=testfile1 +if [ ! -f "$file" ]; then + dd if=/dev/urandom of="$file" count=10 bs=1M +fi +if [ ! -f "$file.c4gh" ]; then + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" +fi +s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/dataset_rotatekey/ + response="$(curl -s -k -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq | grep -c dataset_rotatekey)" -if [ "$response" -ne 2 ]; then - echo "files for rotatekey test failed to upload" +if [ "$response" -ne 1 ]; then + echo "file for rotatekey test failed to upload" exit 1 fi ## ingest and map files to dataset curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/ingest -curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "dataset_rotatekey/testfile2.c4gh", "user": "test@dummy.org"}' http://api:8080/file/ingest -checkStatus verified 2 +checkStatus verified 1 curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "ROTATE-KEY-01", "filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/accession -curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "ROTATE-KEY-02", "filepath": "dataset_rotatekey/testfile2.c4gh", "user": "test@dummy.org"}' http://api:8080/file/accession -checkStatus ready 2 +checkStatus ready 1 -curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["ROTATE-KEY-01", "ROTATE-KEY-02"], "dataset_id": "KEY-ROTATION-TEST-0001", "user": "test@dummy.org"}' http://api:8080/dataset/create +curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["ROTATE-KEY-01"], "dataset_id": "KEY-ROTATION-TEST-0001", "user": "test@dummy.org"}' http://api:8080/dataset/create checkStatus ready 0 errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') @@ -88,8 +99,7 @@ properties=$( mappings=$( jq -c -n \ '$ARGS.positional' \ - --args "ROTATE-KEY-01" \ - --args "ROTATE-KEY-02" + --args "ROTATE-KEY-01" ) mapping_payload=$( @@ -117,7 +127,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ # check DB for updated key hash in sda.files rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") -if "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -neq 2; +if "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -neq 1; then echo "failed to update the key hash of files" exit 1 @@ -142,8 +152,7 @@ if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ exit 1 fi -# download files with rotated key, concatenate header and archive body, decrypt and chack - +## download file with rotated key, concatenate header and archive body, decrypt and check # get rotated header psql -U postgres -h postgres -d sda -At -c "select header from sda.files where stable_id='ROTATE-KEY-01';" | xxd -r -p > testfile1_rotated.c4gh # get archive file @@ -168,4 +177,63 @@ if [ "$(sha256sum testfile1 | cut -d ' ' -f 1)" != "$(sha256sum testfile1 | cut exit 1 fi +### test for errors ### + +# file is already encrypted with key +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" | jq + +checkErrors "already encrypted with the given rotation c4gh key" +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +# multiple accession_id's per message is not supported +mappings=$( + jq -c -n \ + '$ARGS.positional' \ + --args "ROTATE-KEY-01" \ + --args "ROTATE-KEY-02" +) + +mapping_payload=$( + jq -r -c -n \ + --arg type mapping \ + --arg dataset_id KEY-ROTATION-TEST-0001 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64' +) + +mapping_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "rotatekey" \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" | jq + +checkErrors "multiple accession_id's per message is not supported" +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +# rotation key is deprecated +rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/c4gh-keys/deprecate/$rotateKeyHash")" +if [ "$resp" != "200" ]; then + echo "Error when trying to deprecate rotation public key hash, expected 200 got: $resp" + exit 1 +fi + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" | jq + +checkErrors "rotation key is deprecated" +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + echo "Rotate key integration tests completed successfully" From 34f12e55220e01b8142e11c521539f6d0c87f64e Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 15 Sep 2025 09:08:22 +0200 Subject: [PATCH 085/184] add rotatekey documentation --- sda/cmd/rotatekey/rotatekey.md | 111 +++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 sda/cmd/rotatekey/rotatekey.md diff --git a/sda/cmd/rotatekey/rotatekey.md b/sda/cmd/rotatekey/rotatekey.md new file mode 100644 index 000000000..7558201a3 --- /dev/null +++ b/sda/cmd/rotatekey/rotatekey.md @@ -0,0 +1,111 @@ +# rotatekey Service + +Rotates the crypt4gh encryption key of a file that is mapped to a dataset and stored in the SDA. + +## Service Description + +The rotatekey service re-encrypts the header of a file with the configured target key, and updates the database with the new header and encryption key hash. + +When running, rotatekey reads messages from the `rotatekey_stream` RabbitMQ queue. +For each message, these steps are taken (errors halts progress, the message is Nack'ed, an info-error message is sent and the service moves on to the next message): + +1. The message is validated as valid JSON that matches the "dataset-mapping" schema. +2. A database look-up is performed for the configured target public key hash. If the look-up fails or the key has been deprecated, an error is raised. +3. If the massage contains more than one accession IDs, an error is raised and the message is discarded as the service works on a one file per message basis. +4. The file ID is fetched from the database. +5. The key hash of the c4gh key with which the file is currently encrypted is fetched from the database and compared with the configured target key. +6. If these key hashes differ, the reencrypt service is called to re-encrypt the file header with the target key. +7. The file header entry in the database is updated with the new one. +8. The key hash entry for the database is updated with the new one (target key). +9. A re-verify message is compiled, validated and sent to the archived queue so that it is consumed by verify service. +10. The message is Ack'ed. + +## Communication + +- Rotatekey reads messages from one rabbitmq stream (`rotatekey_stream`) +- Rotatekey reads file information, headers and key hashes from the database and can not be started without a database connection. +- Rotatekey makes grpc calls to reencrypt service for re-encrypting the header with the target public key. +- Rotatekey sends messages to the `archived` queue for consumption by the verify service. + +## Configuration + +There are a number of options that can be set for the rotatekey service. +These settings can be set by mounting a yaml-file at `/config.yaml` with settings. + +ex. + +```yaml +log: + level: "debug" + format: "json" +``` + +They may also be set using environment variables like: + +```bash +export LOG_LEVEL="debug" +export LOG_FORMAT="json" +``` + +### Public Key file settings + +This setting controls which crypt4gh keyfile is loaded. + +- `C4GH_ROTATEPUBKEYPATH`: path to the crypt4gh public key to use for reencrypting file headers. + +### RabbitMQ broker settings + +These settings control how sync connects to the RabbitMQ message broker. + +- `BROKER_HOST`: hostname of the rabbitmq server +- `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) +- `BROKER_QUEUE`: message queue or stream to read messages from (commonly `rotatekey_stream`) +- `BROKER_USER`: username to connect to rabbitmq +- `BROKER_PASSWORD`: password to connect to rabbitmq +- `BROKER_ROUTINGKEY`: routing from a rabbitmq exchange to the rotatekey queue + +### PostgreSQL Database settings + +- `DB_HOST`: hostname for the postgresql database +- `DB_PORT`: database port (commonly 5432) +- `DB_USER`: username for the database +- `DB_PASSWORD`: password for the database +- `DB_DATABASE`: database name +- `DB_SSLMODE`: The TLS encryption policy to use for database connections. Valid options are: + - `disable` + - `allow` + - `prefer` + - `require` + - `verify-ca` + - `verify-full` + + More information is available + [in the postgresql documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) + + Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, + and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set. + +- `DB_CLIENTKEY`: key-file for the database client certificate +- `DB_CLIENTCERT`: database client certificate file +- `DB_CACERT`: Certificate Authority (CA) certificate for the database to use + +### GRPC settings + +- `GRPC_HOST`: Host name of the grpc server +- `GRPC_PORT`: Port number of the grpc server +- `GRPC_CACERT`: Certificate Authority (CA) certificate for validating incoming request +- `GRPC_SERVERCERT`: path to the x509 certificate used by the service +- `GRPC_SERVERKEY`: path to the x509 private key used by the service + + +### Logging settings + +- `LOG_FORMAT` can be set to “json” to get logs in json format. All other values result in text logging +- `LOG_LEVEL` can be set to one of the following, in increasing order of severity: + - `trace` + - `debug` + - `info` + - `warn` (or `warning`) + - `error` + - `fatal` + - `panic` From cc4e90dcf209705d6e3eb065ad633c6e28c6ff57 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 17:08:22 +0200 Subject: [PATCH 086/184] separate logging from nacking function --- sda/cmd/rotatekey/rotatekey.go | 53 ++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index e87a40671..fea7ec0b3 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -92,7 +92,8 @@ func main() { err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), delivered.Body) if err != nil { msg := "validation of incoming message (dataset-mapping) failed" - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -102,7 +103,8 @@ func main() { keyhash, err = getKeyHash() if err != nil { msg := "database lookup of the rotation key failed" - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -113,9 +115,8 @@ func main() { // We expect only one aID per message so that we handle errors and nacks properly. // A different json schema seems like a cleaner solution going forward. if len(message.AccessionIDs) > 1 { - msg := "failed to process message" - err = errors.New("multiple accession_ids per message is not supported") - logAndNack(mq, delivered, msg, err) + log.Errorf("failed to process message, reason: multiple accession_id's per message is not supported") + NackAndSendToErrorQueue(mq, delivered, "failed to process message", "multiple accession_id's per message is not supported") continue } @@ -125,7 +126,8 @@ func main() { fileID, err := db.GetFileIDbyAccessionID(aID) if err != nil { msg := fmt.Sprintf("failed to get file-id for file with accession-id: %s", aID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -134,7 +136,8 @@ func main() { oldKeyHash, err := db.GetKeyHash(fileID) if err != nil { msg := fmt.Sprintf("failed to get keyhash for file with accession-id: %s", aID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -142,8 +145,8 @@ func main() { // Check that the file is not already encrypted with the target key if oldKeyHash == keyhash { msg := fmt.Sprintf("failed to reencrypt file with file-id: %s", fileID) - err = errors.New("already encrypted with the given rotation c4gh key") - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSentToErrorQueue(mq, delivered, msg, "already encrypted with the given rotation c4gh key") continue } @@ -151,13 +154,16 @@ func main() { newHeader, err := reencryptFile(aID) if err != nil { msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } if newHeader == nil { + err := errors.New("reencrypt returned empty header") msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -165,7 +171,8 @@ func main() { // Rotate header in database if err := db.StoreHeader(newHeader, fileID); err != nil { msg := fmt.Sprintf("StoreHeader failed for file-id: %s", fileID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -173,7 +180,8 @@ func main() { // Rotate keyhash if err := db.SetKeyHash(keyhash, fileID); err != nil { msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -182,7 +190,8 @@ func main() { reVerify, err := db.GetReVerificationData(aID) if err != nil { msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -191,7 +200,8 @@ func main() { err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", Conf.Broker.SchemasPath), reVerifyMsg) if err != nil { msg := "Validation of outgoing re-verify message failed" - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -199,14 +209,16 @@ func main() { corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) if err != nil { msg := fmt.Sprintf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } if err := mq.SendMessage(corrID, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" - logAndNack(mq, delivered, msg, err) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) continue } @@ -303,12 +315,11 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { return res.Header, nil } -// Send the message to an error queue so it can be analyzed and then nack message. -func logAndNack(mq *broker.AMQPBroker, delivered amqp091.Delivery, msg string, err error) { - log.Errorf("%s, reason: %v", msg, err) +// Nack message without requeue. Send the message to an error queue so it can be analyzed. +func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, msg, reason string) { infoErrorMessage := broker.InfoError{ Error: msg, - Reason: err.Error(), + Reason: reason, OriginalMessage: string(delivered.Body), } body, _ := json.Marshal(infoErrorMessage) From 4bd6ab2aeadf9269d13824a9a2b9cae002dab16b Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 18:57:22 +0200 Subject: [PATCH 087/184] handle case with already encrypted with key at info level --- .github/integration/tests/sda/70_rotate_key_test.sh | 8 -------- sda/cmd/rotatekey/rotatekey.go | 7 ++++--- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index e664d1082..efde57b2d 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -179,14 +179,6 @@ fi ### test for errors ### -# file is already encrypted with key -curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ - -H 'Content-Type: application/json;charset=UTF-8' \ - -d "$mapping_body" | jq - -checkErrors "already encrypted with the given rotation c4gh key" -errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') - # multiple accession_id's per message is not supported mappings=$( jq -c -n \ diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index fea7ec0b3..508648359 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -144,9 +144,10 @@ func main() { // Check that the file is not already encrypted with the target key if oldKeyHash == keyhash { - msg := fmt.Sprintf("failed to reencrypt file with file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - NackAndSentToErrorQueue(mq, delivered, msg, "already encrypted with the given rotation c4gh key") + log.Infof("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack following already encrypted with key message") + } continue } From 3785914e0aba159132a6b420fc879ac25f80af04 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 20:19:08 +0200 Subject: [PATCH 088/184] integration test fixes from review suggestions --- .github/integration/tests/sda/70_rotate_key_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index efde57b2d..337885c8a 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -127,7 +127,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ # check DB for updated key hash in sda.files rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") -if "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -neq 1; +if [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -ne 1 ]; then echo "failed to update the key hash of files" exit 1 @@ -172,7 +172,7 @@ if ! cmp -s "testfile1_rotated" "testfile1" ; then exit 1 fi # compare hashes as well -if [ "$(sha256sum testfile1 | cut -d ' ' -f 1)" != "$(sha256sum testfile1 | cut -d ' ' -f 1)" ]; then +if [ "$(sha256sum testfile1 | cut -d ' ' -f 1)" != "$(sha256sum testfile1_rotated | cut -d ' ' -f 1)" ]; then echo "downloaded file has different sha256 hash from the original one" exit 1 fi From 855f79439315a93ee94d2e2597bb91bff164bafd Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 20:50:59 +0200 Subject: [PATCH 089/184] add CheckKeyHash function --- sda/internal/database/db_functions.go | 20 +++++++++++ sda/internal/database/db_functions_test.go | 41 ++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 61d41bf4e..e00d3a90e 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1070,6 +1070,26 @@ func (dbs *SDAdb) DeprecateKeyHash(keyHash string) error { return nil } +// Check that a key hash exists in the database +func (dbs *SDAdb) CheckKeyHash(keyhash string) error { + hashes, err := dbs.ListKeyHashes() + if err != nil { + return err + } + + for n := range hashes { + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt == "" { + return nil + } + + if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt != "" { + return errors.New("the c4gh key hash has been deprecated") + } + } + + return errors.New("the c4gh key hash is not registered") +} + // ListDatasets lists all datasets as well as the status func (dbs *SDAdb) ListDatasets() ([]*DatasetInfo, error) { dbs.checkAndReconnectIfNeeded() diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 97cc675fe..10348493d 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -975,6 +975,47 @@ func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { db.Close() } +func (suite *DatabaseTests) TestCheckKeyHash() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + assert.NoError(suite.T(), db.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") + anotherKeyhash := "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99" + assert.NoError(suite.T(), db.AddKeyHash(anotherKeyhash, "this is a another key"), "failed to register key in database") + + err = db.CheckKeyHash(anotherKeyhash) + assert.NoError(suite.T(), err, "failed to verify active key hash lookup") + + db.Close() +} + +func (suite *DatabaseTests) TestCheckKeyHash_keyDeprecated() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + assert.NoError(suite.T(), db.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") + anotherKeyhash := "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99" + assert.NoError(suite.T(), db.AddKeyHash(anotherKeyhash, "this is a another key"), "failed to register key in database") + assert.NoError(suite.T(), db.DeprecateKeyHash(anotherKeyhash), "failure when deprecating keyhash") + + err = db.CheckKeyHash(anotherKeyhash) + assert.ErrorContains(suite.T(), err, "the c4gh key hash has been deprecated") + + db.Close() +} + +func (suite *DatabaseTests) TestCheckKeyHash_keyNonExistent() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + assert.NoError(suite.T(), db.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") + + err = db.CheckKeyHash("somekeyhash") + assert.ErrorContains(suite.T(), err, "the c4gh key hash is not registered") + + db.Close() +} + func (suite *DatabaseTests) TestListDatasets() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) From 1321cdf8780472dd2dd5ca30fe582bdc2e3b1dde Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 22:04:37 +0200 Subject: [PATCH 090/184] use db.CheckKeyHash instead of getKeyHash --- sda/cmd/rotatekey/rotatekey.go | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 508648359..92f7229d7 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -55,7 +55,8 @@ func main() { } // Check that key is registered in the db at startup - keyhash, err := getKeyHash() + keyhash := hex.EncodeToString(publicKey[:]) + err = db.CheckKeyHash(keyhash) if err != nil { log.Fatalf("database lookup of the rotation key failed, reason: %v", err) } @@ -100,7 +101,8 @@ func main() { // Fetch rotate key hash before starting work so that we make sure the hash state // has not changed since the application startup. - keyhash, err = getKeyHash() + keyhash := hex.EncodeToString(publicKey[:]) + err = db.CheckKeyHash(keyhash) if err != nil { msg := "database lookup of the rotation key failed" log.Errorf("%s, reason: %v", msg, err) @@ -256,32 +258,6 @@ func reencryptFile(stableID string) ([]byte, error) { return newHeader, nil } -// Check that the key hash exists in the database -func getKeyHash() (string, error) { - keyhash := hex.EncodeToString(publicKey[:]) - hashes, err := db.ListKeyHashes() - if err != nil { - return "", err - } - found := false - for n := range hashes { - if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt != "" { - return "", errors.New("the c4gh key hash has been deprecated") - } - - if hashes[n].Hash == keyhash && hashes[n].DeprecatedAt == "" { - found = true - - break - } - } - if !found { - return "", errors.New("the c4gh key hash is not registered") - } - - return keyhash, nil -} - // reencryptHeader re-encrypts the header of a file using the public key // provided in the request header and returns the new header. The function uses // gRPC to communicate with the re-encrypt service and handles TLS configuration From 64460ea4caedce780c5c66733a4df34e89fc5ddb Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 17 Sep 2025 23:34:20 +0200 Subject: [PATCH 091/184] get file corrID from delivered message instead of fetching it from the db at runtime --- .github/integration/tests/sda/70_rotate_key_test.sh | 8 ++++++++ sda/cmd/rotatekey/rotatekey.go | 11 +---------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 337885c8a..ccf348026 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -88,9 +88,17 @@ checkStatus ready 0 errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') ## trigger key rotation +corrID=$( + curl -s -X POST \ + -H "content-type:application/json" \ + -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ + -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id + ) + properties=$( jq -c -n \ --argjson delivery_mode 2 \ + --arg correlation_id "$corrID" \ --arg content_encoding UTF-8 \ --arg content_type application/json \ '$ARGS.named' diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 92f7229d7..613364e69 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -209,16 +209,7 @@ func main() { continue } - corrID, err := db.GetCorrID(reVerify.User, reVerify.FilePath, aID) - if err != nil { - msg := fmt.Sprintf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) - log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) - - continue - } - - if err := mq.SendMessage(corrID, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) From 6968dceadffcc2ebd1110030c8e6b3bc5db66294 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 18 Sep 2025 18:22:59 +0200 Subject: [PATCH 092/184] add rotatekey-key json schema --- sda/internal/schema/schema.go | 7 +++++++ sda/internal/schema/schema_test.go | 18 +++++++++++++++++ sda/schemas/federated/rotate-key.json | 29 +++++++++++++++++++++++++++ sda/schemas/isolated/rotate-key.json | 29 +++++++++++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 sda/schemas/federated/rotate-key.json create mode 100644 sda/schemas/isolated/rotate-key.json diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index f249ed52f..c878a6eff 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -67,6 +67,8 @@ func getStructName(path string) any { return new(SyncDataset) case "metadata-sync": return new(SyncMetadata) + case "rotate-key": + return new(KeyRotation) default: return "" } @@ -185,3 +187,8 @@ type C4ghPubKey struct { PubKey string `json:"pubkey"` Description string `json:"description"` } + +type KeyRotation struct { + Type string `json:"type"` + FileID string `json:"file_id"` +} diff --git a/sda/internal/schema/schema_test.go b/sda/internal/schema/schema_test.go index bb36014f7..c08c95b01 100644 --- a/sda/internal/schema/schema_test.go +++ b/sda/internal/schema/schema_test.go @@ -469,3 +469,21 @@ func TestValidateJSONBigpictureMetadtaSync(t *testing.T) { msg, _ = json.Marshal(badMsg) assert.Error(t, ValidateJSON(fmt.Sprintf("%s/bigpicture/metadata-sync.json", schemaPath), msg)) } + +func TestValidateJSONKeyRotation(t *testing.T) { + okMsg := KeyRotation{ + Type: "key_rotation", + FileID: "cd532362-e06e-4460-8490-b9ce64b8d9e7", + } + + msg, _ := json.Marshal(okMsg) + assert.Nil(t, ValidateJSON(fmt.Sprintf("%s/isolated/rotate-key.json", schemaPath), msg)) + + badMsg := KeyRotation{ + Type: "foo", + FileID: "cd532362-e06e-4460-8490-b9ce64b8d9e7", + } + + msg, _ = json.Marshal(badMsg) + assert.Error(t, ValidateJSON(fmt.Sprintf("%s/isolated/rotate-key.json", schemaPath), msg)) +} diff --git a/sda/schemas/federated/rotate-key.json b/sda/schemas/federated/rotate-key.json new file mode 100644 index 000000000..cec407ca9 --- /dev/null +++ b/sda/schemas/federated/rotate-key.json @@ -0,0 +1,29 @@ +{ + "title": "JSON schema for SDA key rotation message interface", + "$id": "https://github.com/neicnordic/sensitive-data-archive/tree/master/sda/schemas/federated/rotate-key.json", + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "type", + "file_id" + ], + "additionalProperties": true, + "properties": { + "type": { + "$id": "#/properties/type", + "type": "string", + "title": "The message type", + "description": "The message type", + "const": "key_rotation" + }, + "file_id": { + "$id": "#/properties/file_id", + "type": "string", + "title": "The unique file identifier", + "description": "The unique file identifier", + "examples": [ + "420420cc43-e060-4583-a891-9f8170ee66c8" + ] + } + } +} diff --git a/sda/schemas/isolated/rotate-key.json b/sda/schemas/isolated/rotate-key.json new file mode 100644 index 000000000..6d6bc39d5 --- /dev/null +++ b/sda/schemas/isolated/rotate-key.json @@ -0,0 +1,29 @@ +{ + "title": "JSON schema for SDA key rotation message interface", + "$id": "https://github.com/neicnordic/sensitive-data-archive/tree/master/sda/schemas/isolated/rotate-key.json", + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "required": [ + "type", + "file_id" + ], + "additionalProperties": true, + "properties": { + "type": { + "$id": "#/properties/type", + "type": "string", + "title": "The message type", + "description": "The message type", + "const": "key_rotation" + }, + "file_id": { + "$id": "#/properties/file_id", + "type": "string", + "title": "The unique file identifier", + "description": "The unique file identifier", + "examples": [ + "420420cc43-e060-4583-a891-9f8170ee66c8" + ] + } + } +} From 68ed6a75c0b68c047ecf076f2778c488b4b581bb Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 18 Sep 2025 18:24:06 +0200 Subject: [PATCH 093/184] add GetAccessionID db function --- sda/internal/database/db_functions.go | 31 ++++++++++++ sda/internal/database/db_functions_test.go | 55 ++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index e00d3a90e..46e9c3ebe 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -470,6 +470,37 @@ func (dbs *SDAdb) setAccessionID(accessionID, fileID string) error { return nil } +// GetAccessionID returns the stable id of a file identified by its file_id +func (dbs *SDAdb) GetAccessionID(fileID string) (string, error) { + var ( + aID string + err error + ) + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + aID, err = dbs.getAccessionID(fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return aID, err +} +func (dbs *SDAdb) getAccessionID(fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getAccessionID = "SELECT stable_id FROM sda.files WHERE id = $1;" + var aID string + err := db.QueryRow(getAccessionID, fileID).Scan(&aID) + if err != nil { + return "", err + } + + return aID, nil +} + // MapFilesToDataset maps a set of files to a dataset in the database func (dbs *SDAdb) MapFilesToDataset(datasetID string, accessionIDs []string) error { var err error diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 10348493d..e4fe2af89 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -284,6 +284,61 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { db.Close() } +func (suite *DatabaseTests) TestGetAccessionID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") + err = db.SetVerified(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + stableID := "TEST:000-1234-4567" + err = db.SetAccessionID(stableID, fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + res, err := db.GetAccessionID(fileID) + assert.NoError(suite.T(), err, "got (%v) when getting accessionID of file", err) + assert.Equal(suite.T(), stableID, res, "retrieved accessionID is wrong") + + db.Close() +} + +func (suite *DatabaseTests) TestGetAccessionID_wrongFileID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") + err = db.SetVerified(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + stableID := "TEST:000-1234-4567" + err = db.SetAccessionID(stableID, fileID) + assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) + + // locally reduce RetryTimes to avoid 30s waiting limit of testsuite + RetryTimes = 2 + + // check for bad format + _, err = db.GetAccessionID("someFileID") + assert.ErrorContains(suite.T(), err, "invalid input syntax for type uuid") + + // check for non-existent fileID + _, err = db.GetAccessionID(uuid.New().String()) + assert.ErrorContains(suite.T(), err, "no rows in result set") + + db.Close() +} + func (suite *DatabaseTests) TestGetFileInfo() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) From a96b12ed382cdd5882794473d9e65c43d646dfaa Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 18 Sep 2025 22:42:12 +0200 Subject: [PATCH 094/184] rotatekey consumes rotate-key messages - use fileID as unique file identifier - remove obsolete db funcion - update integration tests --- .../tests/sda/70_rotate_key_test.sh | 76 ++++++++----------- sda/cmd/rotatekey/rotatekey.go | 61 +++++++-------- sda/internal/database/db_functions.go | 28 ------- sda/internal/database/db_functions_test.go | 34 --------- 4 files changed, 58 insertions(+), 141 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index ccf348026..d6c6dbe93 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -94,6 +94,7 @@ corrID=$( -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id ) +fileID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where stable_id='ROTATE-KEY-01';") properties=$( jq -c -n \ @@ -104,34 +105,27 @@ properties=$( '$ARGS.named' ) -mappings=$( - jq -c -n \ - '$ARGS.positional' \ - --args "ROTATE-KEY-01" -) - -mapping_payload=$( +rotatekey_payload=$( jq -r -c -n \ - --arg type mapping \ - --arg dataset_id KEY-ROTATION-TEST-0001 \ - --argjson accession_ids "$mappings" \ + --arg type "key_rotation" \ + --arg file_id "$fileID" \ '$ARGS.named|@base64' ) -mapping_body=$( +rotatekey_body=$( jq -c -n \ --arg vhost test \ --arg name sda \ --argjson properties "$properties" \ --arg routing_key "rotatekey" \ --arg payload_encoding base64 \ - --arg payload "$mapping_payload" \ + --arg payload "$rotatekey_payload" \ '$ARGS.named' ) curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ -H 'Content-Type: application/json;charset=UTF-8' \ - -d "$mapping_body" | jq + -d "$rotatekey_body" | jq # check DB for updated key hash in sda.files rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") @@ -161,11 +155,14 @@ if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ fi ## download file with rotated key, concatenate header and archive body, decrypt and check + # get rotated header psql -U postgres -h postgres -d sda -At -c "select header from sda.files where stable_id='ROTATE-KEY-01';" | xxd -r -p > testfile1_rotated.c4gh + # get archive file archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where stable_id='ROTATE-KEY-01';") s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force + # concatenate and decrypt cat testfile1_rotated.c4gh "$archivePath" > tmp_file && mv tmp_file testfile1_rotated.c4gh C4GH_PASSPHRASE=rotatekeyPass ./crypt4gh decrypt -f testfile1_rotated.c4gh -s rotatekey.sec.pem @@ -187,53 +184,44 @@ fi ### test for errors ### -# multiple accession_id's per message is not supported -mappings=$( - jq -c -n \ - '$ARGS.positional' \ - --args "ROTATE-KEY-01" \ - --args "ROTATE-KEY-02" -) +# rotation key is deprecated +rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/c4gh-keys/deprecate/$rotateKeyHash")" +if [ "$resp" != "200" ]; then + echo "Error when trying to deprecate rotation public key hash, expected 200 got: $resp" + exit 1 +fi -mapping_payload=$( +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$rotatekey_body" | jq + +checkErrors "rotation key is deprecated" +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +# bad message + +rotatekey_payload=$( jq -r -c -n \ - --arg type mapping \ - --arg dataset_id KEY-ROTATION-TEST-0001 \ - --argjson accession_ids "$mappings" \ + --arg type "key_rotation" \ '$ARGS.named|@base64' ) -mapping_body=$( +rotatekey_body=$( jq -c -n \ --arg vhost test \ --arg name sda \ --argjson properties "$properties" \ --arg routing_key "rotatekey" \ --arg payload_encoding base64 \ - --arg payload "$mapping_payload" \ + --arg payload "$rotatekey_payload" \ '$ARGS.named' ) curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ -H 'Content-Type: application/json;charset=UTF-8' \ - -d "$mapping_body" | jq - -checkErrors "multiple accession_id's per message is not supported" -errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') - -# rotation key is deprecated -rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/c4gh-keys/deprecate/$rotateKeyHash")" -if [ "$resp" != "200" ]; then - echo "Error when trying to deprecate rotation public key hash, expected 200 got: $resp" - exit 1 -fi - -curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ - -H 'Content-Type: application/json;charset=UTF-8' \ - -d "$mapping_body" | jq + -d "$rotatekey_body" | jq -checkErrors "rotation key is deprecated" -errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') +checkErrors "validation of incoming message (rotate-key) failed" echo "Rotate key integration tests completed successfully" diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 613364e69..04fece62f 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -1,5 +1,5 @@ -// The rotatekey service accepts messages for files mapped to a dataset, -// re-encrypts their header with a configured public key and stores it +// The rotatekey service accepts messages to re-encrypt a file identified by its fileID. +// The service re-encrypts the file header with a configured public key and stores it // in the database together with the key-hash of the rotation key. // I then sends a message to verify so the file is re-verified. @@ -78,7 +78,7 @@ func main() { }() log.Info("Starting rotatekey service") - var message schema.DatasetMapping + var message schema.KeyRotation go func() { messages, err := mq.GetMessages(Conf.Broker.Queue) @@ -90,9 +90,9 @@ func main() { delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", Conf.Broker.SchemasPath), delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), delivered.Body) if err != nil { - msg := "validation of incoming message (dataset-mapping) failed" + msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) @@ -114,30 +114,12 @@ func main() { // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - // We expect only one aID per message so that we handle errors and nacks properly. - // A different json schema seems like a cleaner solution going forward. - if len(message.AccessionIDs) > 1 { - log.Errorf("failed to process message, reason: multiple accession_id's per message is not supported") - NackAndSendToErrorQueue(mq, delivered, "failed to process message", "multiple accession_id's per message is not supported") - - continue - } - - aID := message.AccessionIDs[0] - - fileID, err := db.GetFileIDbyAccessionID(aID) - if err != nil { - msg := fmt.Sprintf("failed to get file-id for file with accession-id: %s", aID) - log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) - - continue - } + fileID := message.FileID // Get current keyhash for the file, send to error queue if this fails oldKeyHash, err := db.GetKeyHash(fileID) if err != nil { - msg := fmt.Sprintf("failed to get keyhash for file with accession-id: %s", aID) + msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) @@ -154,9 +136,9 @@ func main() { continue } - newHeader, err := reencryptFile(aID) + newHeader, err := reencryptFile(fileID) if err != nil { - msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) @@ -164,7 +146,7 @@ func main() { } if newHeader == nil { err := errors.New("reencrypt returned empty header") - msg := fmt.Sprintf("failed to rotate c4gh key for file %s", aID) + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) @@ -189,6 +171,15 @@ func main() { continue } + aID, err := db.GetAccessionID(fileID) + if err != nil { + msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + + continue + } + // Send re-verify message reVerify, err := db.GetReVerificationData(aID) if err != nil { @@ -226,10 +217,10 @@ func main() { <-forever } -func reencryptFile(stableID string) ([]byte, error) { - log.Debugf("rotating c4gh key for file with stable-id: %s", stableID) +func reencryptFile(fileID string) ([]byte, error) { + log.Debugf("rotating c4gh key for file with file-id: %s", fileID) - header, err := db.GetHeaderForStableID(stableID) + header, err := db.GetHeader(fileID) if err != nil { return nil, err } @@ -250,10 +241,10 @@ func reencryptFile(stableID string) ([]byte, error) { } // reencryptHeader re-encrypts the header of a file using the public key -// provided in the request header and returns the new header. The function uses -// gRPC to communicate with the re-encrypt service and handles TLS configuration -// if needed. The function also handles the case where the CA certificate is -// provided for secure communication. +// provided and returns the new header. The function uses gRPC to +// communicate with the re-encrypt service and handles TLS configuration +// if needed. The function also handles the case where the CA certificate +// is provided for secure communication. func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { var opts []grpc.DialOption switch { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 46e9c3ebe..34942cf4d 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -63,34 +63,6 @@ func (dbs *SDAdb) getFileID(corrID string) (string, error) { return fileID, nil } -func (dbs *SDAdb) GetFileIDbyAccessionID(accessionID string) (string, error) { - var ( - err error - count int - ID string - ) - - for count == 0 || (err != nil && count < RetryTimes) { - ID, err = dbs.getFileIDbyAccessionID(accessionID) - count++ - } - - return ID, err -} -func (dbs *SDAdb) getFileIDbyAccessionID(accessionID string) (string, error) { - dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const getFileID = "SELECT id FROM sda.files where stable_id = $1;" - - var fileID string - err := db.QueryRow(getFileID, accessionID).Scan(&fileID) - if err != nil { - return "", err - } - - return fileID, nil -} - // GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID // and that is not yet archived func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, error) { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index e4fe2af89..54e989518 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -59,40 +59,6 @@ func (suite *DatabaseTests) TestGetFileID() { db.Close() } -func (suite *DatabaseTests) TestGetFileIDbyAccessionID() { - db, err := NewSDAdb(suite.dbConf) - assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - - // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") - assert.NoError(suite.T(), err, "failed to register file in database") - stableID := "TEST:000-1234-4567" - err = db.SetAccessionID(stableID, fileID) - assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - - retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) - assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - assert.Equal(suite.T(), fileID, retrievedFileID) - - db.Close() -} - -func (suite *DatabaseTests) TestGetFileIDbyAccessionID_nonexistentID() { - db, err := NewSDAdb(suite.dbConf) - assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - - // register a file in the database - _, err = db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") - assert.NoError(suite.T(), err, "failed to register file in database") - - stableID := "TEST:000-1234-4567" - retrievedFileID, err := db.GetFileIDbyAccessionID(stableID) - assert.ErrorContains(suite.T(), err, "no rows in result set") - assert.Equal(suite.T(), "", retrievedFileID) - - db.Close() -} - func (suite *DatabaseTests) TestUpdateFileEventLog() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) From 26b866e4d31469433ad364ad4a69ec0a019be4b6 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 19 Sep 2025 00:18:55 +0200 Subject: [PATCH 095/184] remove rotatekey_stream --- rabbitmq/definitions.json | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/rabbitmq/definitions.json b/rabbitmq/definitions.json index f19aff92a..acb5e0c0d 100644 --- a/rabbitmq/definitions.json +++ b/rabbitmq/definitions.json @@ -54,21 +54,6 @@ "src-uri": "amqp:///sda" }, "vhost": "sda" - }, - { - "component": "shovel", - "name": "rotatekey", - "value": { - "ack-mode": "on-confirm", - "dest-queue": "rotatekey", - "dest-protocol": "amqp091", - "dest-uri": "amqp:///sda", - "src-delete-after": "never", - "src-protocol": "amqp091", - "src-queue": "rotatekey_stream", - "src-uri": "amqp:///sda" - }, - "vhost": "sda" } ], "global_parameters": [], @@ -171,16 +156,6 @@ "auto_delete": false, "arguments": {} }, - { - "name": "rotatekey_stream", - "vhost": "sda", - "durable": true, - "auto_delete": false, - "arguments": { - "x-max-age": "1M", - "x-queue-type": "stream" - } - }, { "name": "catch_all.dead", "vhost": "sda", @@ -294,7 +269,7 @@ "vhost": "sda", "destination_type": "queue", "arguments": {}, - "destination": "rotatekey_stream", + "destination": "rotatekey", "routing_key": "rotatekey" }, { From 3bdd1d866ea59eeda22c3791d5cc559a67d9f11e Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 19 Sep 2025 22:11:43 +0200 Subject: [PATCH 096/184] add utility function CallReencryptHeader to reencrypt - CallReencryptHeader wraps ReencryptHeader - add unittests --- sda/cmd/reencrypt/reencrypt_test.go | 142 ++++++++++++++++++++++ sda/internal/reencrypt/reencrypt_utils.go | 48 ++++++++ 2 files changed, 190 insertions(+) create mode 100644 sda/internal/reencrypt/reencrypt_utils.go diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 779217454..897fa450d 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -342,3 +342,145 @@ func (ts *ReEncryptTests) TestReencryptHeader_TLS() { assert.NoError(ts.T(), err) assert.Equal(ts.T(), "content", string(data)) } + +func (ts *ReEncryptTests) TestCallReencryptHeader() { + lis, err := net.Listen("tcp", "localhost:50061") + if err != nil { + ts.T().FailNow() + } + + go func() { + var opts []grpc.ServerOption + s := grpc.NewServer(opts...) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) + if err := s.Serve(lis); err != nil { + ts.T().Fail() + } + }() + + grpcConf := config.Grpc{ + Host: "localhost", + Port: 50061, + Timeout: 30, + } + res, err := re.CallReencryptHeader(ts.FileHeader, ts.UserPubKeyString, grpcConf) + assert.NoError(ts.T(), err) + + assert.Equal(ts.T(), "crypt4gh", string(res[:8])) + + hr := bytes.NewReader(res) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) + + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) + + data, err := io.ReadAll(c4gh) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) +} + +func (ts *ReEncryptTests) TestCallReencryptHeaderTLS() { + certPath := ts.T().TempDir() + helper.MakeCerts(certPath) + rootCAs := x509.NewCertPool() + cacertFile, err := os.ReadFile(certPath + "/ca.crt") + if err != nil { + ts.T().FailNow() + } + ok := rootCAs.AppendCertsFromPEM(cacertFile) + if !ok { + ts.T().FailNow() + } + certs, err := tls.LoadX509KeyPair(certPath+"/tls.crt", certPath+"/tls.key") + if err != nil { + ts.T().Log(err.Error()) + ts.T().FailNow() + } + + lis, err := net.Listen("tcp", "localhost:50062") + if err != nil { + ts.T().FailNow() + } + + go func() { + serverCreds := credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + ClientAuth: tls.RequireAndVerifyClientCert, + MinVersion: tls.VersionTLS13, + ClientCAs: rootCAs, + }, + ) + opts := []grpc.ServerOption{grpc.Creds(serverCreds)} + s := grpc.NewServer(opts...) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) + if err := s.Serve(lis); err != nil { + ts.T().Fail() + } + }() + + clientCreds := credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + RootCAs: rootCAs, + }, + ) + + grpcConf := config.Grpc{ + ClientCreds: clientCreds, + Host: "localhost", + Port: 50062, + Timeout: 30, + } + res, err := re.CallReencryptHeader(ts.FileHeader, ts.UserPubKeyString, grpcConf) + assert.NoError(ts.T(), err) + + assert.Equal(ts.T(), "crypt4gh", string(res[:8])) + + hr := bytes.NewReader(res) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) + + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) + + data, err := io.ReadAll(c4gh) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) +} + +func (ts *ReEncryptTests) TestCallReencryptHeader_ConnectionError() { + grpcConf := config.Grpc{ + Host: "locahost", + Port: 50063, + Timeout: 30, + } + _, err := re.CallReencryptHeader(ts.FileHeader, ts.UserPubKeyString, grpcConf) + assert.Error(ts.T(), err, "expected a connection error") +} + +func (ts *ReEncryptTests) TestCallReencryptHeader_BadInput() { + lis, err := net.Listen("tcp", "localhost:50064") + if err != nil { + ts.T().FailNow() + } + + go func() { + var opts []grpc.ServerOption + s := grpc.NewServer(opts...) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) + if err := s.Serve(lis); err != nil { + ts.T().Fail() + } + }() + + grpcConf := config.Grpc{ + Host: "localhost", + Port: 50064, + Timeout: 30, + } + + res, err := re.CallReencryptHeader(ts.FileHeader, "somekey", grpcConf) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), res) +} diff --git a/sda/internal/reencrypt/reencrypt_utils.go b/sda/internal/reencrypt/reencrypt_utils.go new file mode 100644 index 000000000..62ca7b794 --- /dev/null +++ b/sda/internal/reencrypt/reencrypt_utils.go @@ -0,0 +1,48 @@ +package reencrypt + +import ( + "context" + "fmt" + "time" + + "github.com/neicnordic/sensitive-data-archive/internal/config" + log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// CallReencryptHeader re-encrypts the header of a file using the public key +// provided and returns the new header. The function uses gRPC to +// communicate with the re-encrypt service and handles TLS configuration +// if needed. The function also handles the case where the CA certificate +// is provided for secure communication. +func CallReencryptHeader(oldHeader []byte, c4ghPubKey string, grpcConf config.Grpc) ([]byte, error) { + var opts []grpc.DialOption + switch { + case grpcConf.ClientCreds != nil: + opts = append(opts, grpc.WithTransportCredentials(grpcConf.ClientCreds)) + default: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", grpcConf.Host, grpcConf.Port), opts...) + if err != nil { + log.Errorf("failed to open a new gRPC channel, reason: %v", err) + + return nil, err + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(grpcConf.Timeout)*time.Second) + defer cancel() + + c := NewReencryptClient(conn) + res, err := c.ReencryptHeader(ctx, &ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) + if err != nil { + log.Errorf("failed to connect to the reencrypt service, reason %v", err) + + return nil, err + } + + return res.Header, nil +} From a46b25cd3bfcf95897b27b274c87e3ce52bb2d51 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 19 Sep 2025 22:16:34 +0200 Subject: [PATCH 097/184] replace reencrypt logic with CallReencryptHeader --- sda/cmd/rotatekey/rotatekey.go | 41 ++-------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 04fece62f..cf91008b8 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -7,13 +7,11 @@ package main import ( "bytes" - "context" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" - "time" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -23,8 +21,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) var ( @@ -232,46 +228,13 @@ func reencryptFile(fileID string) ([]byte, error) { } pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) - newHeader, err := reencryptHeader(header, pubKeyEncoded) - if err != nil { - return nil, err - } - - return newHeader, nil -} - -// reencryptHeader re-encrypts the header of a file using the public key -// provided and returns the new header. The function uses gRPC to -// communicate with the re-encrypt service and handles TLS configuration -// if needed. The function also handles the case where the CA certificate -// is provided for secure communication. -func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { - var opts []grpc.DialOption - switch { - case Conf.RotateKey.Grpc.ClientCreds != nil: - opts = append(opts, grpc.WithTransportCredentials(Conf.RotateKey.Grpc.ClientCreds)) - default: - opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) - } + newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, Conf.RotateKey.Grpc) - conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", Conf.RotateKey.Grpc.Host, Conf.RotateKey.Grpc.Port), opts...) if err != nil { - log.Errorf("failed to connect to the reencrypt service, reason: %s", err) - return nil, err } - defer conn.Close() - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Conf.RotateKey.Grpc.Timeout)*time.Second) - defer cancel() - c := reencrypt.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) - if err != nil { - return nil, err - } - - return res.Header, nil + return newHeader, nil } // Nack message without requeue. Send the message to an error queue so it can be analyzed. From 769ab4879fe0d608fa5db8b1eee40d9b3b24afd1 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Sun, 21 Sep 2025 15:50:16 +0200 Subject: [PATCH 098/184] terminate app if target key is deprecated at runtime --- .../tests/sda/70_rotate_key_test.sh | 53 ++++++++++++++++--- sda/cmd/rotatekey/rotatekey.go | 7 +-- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index d6c6dbe93..51de6337d 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -33,6 +33,19 @@ checkErrors() { done } +checkConsumers() { + RETRY_TIMES=0 + until [ "$(curl -su guest:guest http://localhost:15672/api/consumers | jq '.[].queue.name' | grep -c "$1")" -eq "$2" ]; do + echo "waiting for $1 consumer status" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for $1 consumer status" + exit 1 + fi + sleep 2 + done +} + # cleanup queues and database URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then @@ -184,7 +197,9 @@ fi ### test for errors ### -# rotation key is deprecated +## test rotation key is deprecated during runtime +echo "test rotation key is deprecated during runtime" + rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST "http://api:8080/c4gh-keys/deprecate/$rotateKeyHash")" if [ "$resp" != "200" ]; then @@ -192,16 +207,40 @@ if [ "$resp" != "200" ]; then exit 1 fi +rotatekey_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "rotatekey" \ + --arg payload_encoding base64 \ + --arg payload "$rotatekey_payload" \ + '$ARGS.named' +) + curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ -H 'Content-Type: application/json;charset=UTF-8' \ -d "$rotatekey_body" | jq -checkErrors "rotation key is deprecated" -errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') +# check that app failed +checkConsumers rotatekey 0 -# bad message +## test app attempts to start with a configured rotation key that is deprecated +echo "test app fails to start with a configured rotation key that is invalid" -rotatekey_payload=$( +sleep 2 +# app will keep failing until we restore tha target key as active +checkConsumers rotatekey 0 +deprecationDate=$(psql -U postgres -h postgres -d sda -At -c "select deprecated_at from sda.encryption_keys where deprecated_at is not null;") +psql -U postgres -h postgres -d sda -At -c "UPDATE sda.encryption_keys SET deprecated_at = null WHERE deprecated_at = '$deprecationDate';" + +# check that app recovered when it found a valid target key +checkConsumers rotatekey 1 + +## test bad message +test "test bad mq message" + +rotatekey_payload_bad=$( jq -r -c -n \ --arg type "key_rotation" \ '$ARGS.named|@base64' @@ -214,7 +253,7 @@ rotatekey_body=$( --argjson properties "$properties" \ --arg routing_key "rotatekey" \ --arg payload_encoding base64 \ - --arg payload "$rotatekey_payload" \ + --arg payload "$rotatekey_payload_bad" \ '$ARGS.named' ) @@ -224,4 +263,4 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ checkErrors "validation of incoming message (rotate-key) failed" -echo "Rotate key integration tests completed successfully" +printf "\033[32mRotate key integration tests completed successfully\033[0m\n" diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index cf91008b8..089a3a060 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -99,12 +99,9 @@ func main() { // has not changed since the application startup. keyhash := hex.EncodeToString(publicKey[:]) err = db.CheckKeyHash(keyhash) + // exit app if target key was modified after app start-up, e.g. if key has been deprecated if err != nil { - msg := "database lookup of the rotation key failed" - log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) - - continue + log.Fatalf("check of target key failed, reason: %v", err) } // we unmarshal the message in the validation step so this is safe to do From 5d59aa5cb6872e052e3c9f88d7248440cfced9ee Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 22 Sep 2025 14:48:22 +0200 Subject: [PATCH 099/184] refactoring suggestions frm review - move reencryotFile function logic to main - encode pubkey at startup - avoid using global variables --- sda/cmd/rotatekey/rotatekey.go | 82 +++++++++++++++------------------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 089a3a060..a7e7305d1 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -23,16 +23,9 @@ import ( log "github.com/sirupsen/logrus" ) -var ( - err error - publicKey *[32]byte - db *database.SDAdb - Conf *config.Config -) - func main() { forever := make(chan bool) - Conf, err = config.NewConfig("rotatekey") + Conf, err := config.NewConfig("rotatekey") if err != nil { log.Fatal(err) } @@ -40,16 +33,23 @@ func main() { if err != nil { log.Fatal(err) } - db, err = database.NewSDAdb(Conf.Database) + db, err := database.NewSDAdb(Conf.Database) if err != nil { log.Fatal(err) } - publicKey, err = config.GetC4GHPublicKey("rotatekey") + publicKey, err := config.GetC4GHPublicKey("rotatekey") if err != nil { log.Fatal(err) } + // encode pubkey as pem and then as base64 string + tmp := &bytes.Buffer{} + if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *publicKey); err != nil { + log.Fatal(err) + } + pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) + // Check that key is registered in the db at startup keyhash := hex.EncodeToString(publicKey[:]) err = db.CheckKeyHash(keyhash) @@ -90,7 +90,7 @@ func main() { if err != nil { msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -114,7 +114,7 @@ func main() { if err != nil { msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -129,11 +129,23 @@ func main() { continue } - newHeader, err := reencryptFile(fileID) + // reencrypt header + log.Debugf("rotating c4gh key for file with file-id: %s", fileID) + + header, err := db.GetHeader(fileID) + if err != nil { + msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + + continue + } + + newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, Conf.RotateKey.Grpc) if err != nil { msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -141,7 +153,7 @@ func main() { err := errors.New("reencrypt returned empty header") msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -150,7 +162,7 @@ func main() { if err := db.StoreHeader(newHeader, fileID); err != nil { msg := fmt.Sprintf("StoreHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -159,7 +171,7 @@ func main() { if err := db.SetKeyHash(keyhash, fileID); err != nil { msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -168,7 +180,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -178,7 +190,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -188,7 +200,7 @@ func main() { if err != nil { msg := "Validation of outgoing re-verify message failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -196,7 +208,7 @@ func main() { if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) continue } @@ -210,32 +222,8 @@ func main() { <-forever } -func reencryptFile(fileID string) ([]byte, error) { - log.Debugf("rotating c4gh key for file with file-id: %s", fileID) - - header, err := db.GetHeader(fileID) - if err != nil { - return nil, err - } - - // encode pubkey as pem and then as base64 string - tmp := &bytes.Buffer{} - if err = keys.WriteCrypt4GHX25519PublicKey(tmp, *publicKey); err != nil { - return nil, err - } - pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) - - newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, Conf.RotateKey.Grpc) - - if err != nil { - return nil, err - } - - return newHeader, nil -} - // Nack message without requeue. Send the message to an error queue so it can be analyzed. -func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, msg, reason string) { +func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, mqExchange, msg, reason string) { infoErrorMessage := broker.InfoError{ Error: msg, Reason: reason, @@ -243,7 +231,7 @@ func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(delivered.CorrelationId, mqExchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { From 07770927753aed1a13080584fd3080a8bac581d8 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 23 Sep 2025 13:49:36 +0200 Subject: [PATCH 100/184] update rotatekey docs with suggestions from review --- sda/cmd/rotatekey/rotatekey.md | 36 +++++++++++++++++----------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.md b/sda/cmd/rotatekey/rotatekey.md index 7558201a3..007d6a4aa 100644 --- a/sda/cmd/rotatekey/rotatekey.md +++ b/sda/cmd/rotatekey/rotatekey.md @@ -1,31 +1,31 @@ # rotatekey Service -Rotates the crypt4gh encryption key of a file that is mapped to a dataset and stored in the SDA. +Rotates the crypt4gh encryption key of ingested file headers. ## Service Description -The rotatekey service re-encrypts the header of a file with the configured target key, and updates the database with the new header and encryption key hash. +The `rotatekey` service re-encrypts the header of a file with the configured target key, and updates the database with the new header and encryption key hash. -When running, rotatekey reads messages from the `rotatekey_stream` RabbitMQ queue. -For each message, these steps are taken (errors halts progress, the message is Nack'ed, an info-error message is sent and the service moves on to the next message): +When running, rotatekey reads messages from the `rotatekey` RabbitMQ queue. +For each message, these steps are taken: -1. The message is validated as valid JSON that matches the "dataset-mapping" schema. -2. A database look-up is performed for the configured target public key hash. If the look-up fails or the key has been deprecated, an error is raised. -3. If the massage contains more than one accession IDs, an error is raised and the message is discarded as the service works on a one file per message basis. -4. The file ID is fetched from the database. -5. The key hash of the c4gh key with which the file is currently encrypted is fetched from the database and compared with the configured target key. -6. If these key hashes differ, the reencrypt service is called to re-encrypt the file header with the target key. -7. The file header entry in the database is updated with the new one. -8. The key hash entry for the database is updated with the new one (target key). -9. A re-verify message is compiled, validated and sent to the archived queue so that it is consumed by verify service. -10. The message is Ack'ed. +1. The message is validated as valid JSON that matches the "rotate-key" schema. +2. A database look-up is performed for the configured target public key hash. If the look-up fails or the key has been deprecated, the service will exit. +3. The key hash of the c4gh key with which the file is currently encrypted is fetched from the database and compared with the configured target key. +4. If these key hashes differ, the reencrypt service is called to re-encrypt the file header with the target key. +5. The file header entry in the database is updated with the new one. +6. The key hash entry in the database is updated with the new one (target key). +7. A re-verify message is compiled, validated and sent to the archived queue so that it is consumed by the `verify` service. +8. The message is Ack'ed. + +In case of any errors during the above process, progress will be halted the message is Nack'ed, an info-error message is sent and the service moves on to the next message. ## Communication -- Rotatekey reads messages from one rabbitmq stream (`rotatekey_stream`) +- Rotatekey reads messages from one rabbitmq queue (`rotatekey`). - Rotatekey reads file information, headers and key hashes from the database and can not be started without a database connection. -- Rotatekey makes grpc calls to reencrypt service for re-encrypting the header with the target public key. -- Rotatekey sends messages to the `archived` queue for consumption by the verify service. +- Rotatekey makes grpc calls to `reencrypt` service for re-encrypting the header with the target public key. +- Rotatekey sends messages to the `archived` queue for consumption by the `verify` service. ## Configuration @@ -85,7 +85,7 @@ These settings control how sync connects to the RabbitMQ message broker. Note that if `DB_SSLMODE` is set to anything but `disable`, then `DB_CACERT` needs to be set, and if set to `verify-full`, then `DB_CLIENTCERT`, and `DB_CLIENTKEY` must also be set. -- `DB_CLIENTKEY`: key-file for the database client certificate +- `DB_CLIENTKEY`: key file for the database client certificate - `DB_CLIENTCERT`: database client certificate file - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use From 6e52c6b637bab0256fdfa9822c8c25a3fa1726cf Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 24 Sep 2025 11:11:02 +0200 Subject: [PATCH 101/184] have the input to GetC4GHPublicKey be the pubkey path - add the pubkey to the sync and rotatekey config directly - update unittests --- sda/internal/config/config.go | 30 +++++++++++++++++------------- sda/internal/config/config_test.go | 25 +++++++++++++++---------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 147030c03..840b7d521 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -72,7 +72,8 @@ type ReEncConfig struct { } type RotateKeyConf struct { - Grpc Grpc + Grpc Grpc + PublicKey *[32]byte } type Sync struct { @@ -82,6 +83,7 @@ type Sync struct { RemotePassword string RemotePort int RemoteUser string + PublicKey *[32]byte } type SyncAPIConf struct { @@ -686,6 +688,11 @@ func NewConfig(app string) (*Config, error) { if err != nil { return nil, err } + + c.RotateKey.PublicKey, err = GetC4GHPublicKey(viper.GetString("c4gh.rotatePubKeyPath")) + if err != nil { + return nil, err + } case "s3inbox": err := c.configBroker() if err != nil { @@ -708,7 +715,8 @@ func NewConfig(app string) (*Config, error) { return nil, err } - if err := c.configDatabase(); err != nil { + err := c.configDatabase() + if err != nil { return nil, err } @@ -1164,6 +1172,12 @@ func (c *Config) configSync() error { c.Sync.RemoteUser = viper.GetString("sync.remote.user") c.Sync.CenterPrefix = viper.GetString("sync.centerPrefix") + var err error + c.Sync.PublicKey, err = GetC4GHPublicKey(viper.GetString("c4gh.syncPubKeyPath")) + if err != nil { + return err + } + return nil } @@ -1234,17 +1248,7 @@ func GetC4GHprivateKeys() ([]*[32]byte, error) { } // GetC4GHPublicKey reads the c4gh public key -func GetC4GHPublicKey(app string) (*[32]byte, error) { - var keyPath string - switch app { - case "sync": - keyPath = viper.GetString("c4gh.syncPubKeyPath") - case "rotatekey": - keyPath = viper.GetString("c4gh.rotatePubKeyPath") - default: - return nil, errors.New("pubKey not set") - } - +func GetC4GHPublicKey(keyPath string) (*[32]byte, error) { // Make sure the key path and passphrase is valid keyFile, err := os.Open(keyPath) if err != nil { diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index a724fd249..04f200389 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -20,6 +20,7 @@ import ( type ConfigTestSuite struct { suite.Suite + pubKeyPath string } var certPath, rootDir string @@ -67,6 +68,11 @@ func (ts *ConfigTestSuite) SetupTest() { viper.Set("inbox.type", "s3") viper.Set("server.jwtpubkeypath", "testpath") viper.Set("log.level", "debug") + + pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" + ts.pubKeyPath, _ = os.MkdirTemp("", "pubkey") + err = os.WriteFile(ts.pubKeyPath+"/c4gh.pub", []byte(pubKey), 0600) + assert.NoError(ts.T(), err) } func (ts *ConfigTestSuite) TearDownTest() { @@ -303,7 +309,7 @@ func (ts *ConfigTestSuite) TestSyncConfig() { viper.Set("sync.remote.password", "test") viper.Set("c4gh.filepath", "/keys/key") viper.Set("c4gh.passphrase", "pass") - viper.Set("c4gh.syncPubKeyPath", "/keys/recipient") + viper.Set("c4gh.syncPubKeyPath", ts.pubKeyPath+"/c4gh.pub") config, err = NewConfig("sync") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -325,6 +331,8 @@ func (ts *ConfigTestSuite) TestSyncConfig() { assert.NotNil(ts.T(), config.Sync) assert.NotNil(ts.T(), config.Sync.Destination.Posix) assert.Equal(ts.T(), "test", config.Sync.Destination.Posix.Location) + + defer os.RemoveAll(ts.pubKeyPath) } func (ts *ConfigTestSuite) TestRotateKeyConfig() { @@ -334,7 +342,7 @@ func (ts *ConfigTestSuite) TestRotateKeyConfig() { assert.Error(ts.T(), err) assert.Nil(ts.T(), config) - viper.Set("c4gh.rotatePubKeyPath", "/keys/recipient") + viper.Set("c4gh.rotatePubKeyPath", ts.pubKeyPath+"/c4gh.pub") config, err = NewConfig("rotatekey") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -353,25 +361,22 @@ func (ts *ConfigTestSuite) TestRotateKeyConfig() { assert.NotNil(ts.T(), config.RotateKey) assert.NotNil(ts.T(), config.RotateKey.Grpc) assert.Equal(ts.T(), "reencrypt", config.RotateKey.Grpc.Host) + + defer os.RemoveAll(ts.pubKeyPath) } func (ts *ConfigTestSuite) TestGetC4GHPublicKey() { - pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" - pubKeyPath, _ := os.MkdirTemp("", "pubkey") - err := os.WriteFile(pubKeyPath+"/c4gh.pub", []byte(pubKey), 0600) - assert.NoError(ts.T(), err) - var kb [32]byte k, _ := base64.StdEncoding.DecodeString("uQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=") copy(kb[:], k) - viper.Set("c4gh.syncPubKeyPath", pubKeyPath+"/c4gh.pub") - pkBytes, err := GetC4GHPublicKey("sync") + viper.Set("c4gh.syncPubKeyPath", ts.pubKeyPath+"/c4gh.pub") + pkBytes, err := GetC4GHPublicKey(ts.pubKeyPath + "/c4gh.pub") assert.NoError(ts.T(), err) assert.NotNil(ts.T(), pkBytes) assert.Equal(ts.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") - defer os.RemoveAll(pubKeyPath) + defer os.RemoveAll(ts.pubKeyPath) } func (ts *ConfigTestSuite) TestGetC4GHKey() { key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" From c110a86bf230874333bd26b85d63ffbdef421488 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 24 Sep 2025 11:17:41 +0200 Subject: [PATCH 102/184] sync: load the pubkey from config instead of calling config.GetC4GHPublicKey --- sda/cmd/sync/sync.go | 9 ++------- sda/cmd/sync/sync_test.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index b8f2fb638..2c7ac2977 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -25,7 +25,7 @@ import ( var ( err error - key, publicKey *[32]byte + key *[32]byte db *database.SDAdb conf *config.Config archive, syncDestination storage.Backend @@ -60,11 +60,6 @@ func main() { log.Fatal(err) } - publicKey, err = config.GetC4GHPublicKey("sync") - if err != nil { - log.Fatal(err) - } - defer mq.Channel.Close() defer mq.Connection.Close() defer db.Close() @@ -196,7 +191,7 @@ func syncFiles(stableID string) error { } pubkeyList := [][chacha20poly1305.KeySize]byte{} - pubkeyList = append(pubkeyList, *publicKey) + pubkeyList = append(pubkeyList, *conf.Sync.PublicKey) newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) if err != nil { return err diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index 95d35a981..fd31473db 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -28,6 +28,7 @@ var dbPort int type SyncTest struct { suite.Suite + keyPath string } func TestSyncTestSuite(t *testing.T) { @@ -135,23 +136,23 @@ func (s *SyncTest) SetupTest() { viper.Set("sync.remote.password", "pass") key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" - keyPath, _ := os.MkdirTemp("", "key") - err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600) + s.keyPath, _ = os.MkdirTemp("", "key") + err := os.WriteFile(s.keyPath+"/c4gh.key", []byte(key), 0600) assert.NoError(s.T(), err) - viper.Set("c4gh.filepath", keyPath+"/c4gh.key") + viper.Set("c4gh.filepath", s.keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" - err = os.WriteFile(keyPath+"/c4gh.pub", []byte(pubKey), 0600) + err = os.WriteFile(s.keyPath+"/c4gh.pub", []byte(pubKey), 0600) assert.NoError(s.T(), err) - viper.Set("c4gh.syncPubKeyPath", keyPath+"/c4gh.pub") - - defer os.RemoveAll(keyPath) + viper.Set("c4gh.syncPubKeyPath", s.keyPath+"/c4gh.pub") } func (s *SyncTest) TestBuildSyncDatasetJSON() { s.SetupTest() + defer os.RemoveAll(s.keyPath) + conf, err := config.NewConfig("sync") assert.NoError(s.T(), err) From ae85b5f2c64f70b9bc6a7ee62b5990a7807740ae Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 24 Sep 2025 11:47:51 +0200 Subject: [PATCH 103/184] rework pub key handling logic - read pubkey during configuration - base64 encode pubkey at startup - do not export config --- sda/cmd/rotatekey/rotatekey.go | 51 +++++++++++++++------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index a7e7305d1..b50a258d8 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -1,7 +1,7 @@ // The rotatekey service accepts messages to re-encrypt a file identified by its fileID. // The service re-encrypts the file header with a configured public key and stores it // in the database together with the key-hash of the rotation key. -// I then sends a message to verify so the file is re-verified. +// It then sends a message to verify so that the file is re-verified. package main @@ -25,33 +25,28 @@ import ( func main() { forever := make(chan bool) - Conf, err := config.NewConfig("rotatekey") + conf, err := config.NewConfig("rotatekey") if err != nil { log.Fatal(err) } - mq, err := broker.NewMQ(Conf.Broker) + mq, err := broker.NewMQ(conf.Broker) if err != nil { log.Fatal(err) } - db, err := database.NewSDAdb(Conf.Database) - if err != nil { - log.Fatal(err) - } - - publicKey, err := config.GetC4GHPublicKey("rotatekey") + db, err := database.NewSDAdb(conf.Database) if err != nil { log.Fatal(err) } // encode pubkey as pem and then as base64 string tmp := &bytes.Buffer{} - if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *publicKey); err != nil { + if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *conf.RotateKey.PublicKey); err != nil { log.Fatal(err) } pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) // Check that key is registered in the db at startup - keyhash := hex.EncodeToString(publicKey[:]) + keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) err = db.CheckKeyHash(keyhash) if err != nil { log.Fatalf("database lookup of the rotation key failed, reason: %v", err) @@ -77,7 +72,7 @@ func main() { var message schema.KeyRotation go func() { - messages, err := mq.GetMessages(Conf.Broker.Queue) + messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { log.Fatal(err) } @@ -86,18 +81,18 @@ func main() { delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } // Fetch rotate key hash before starting work so that we make sure the hash state // has not changed since the application startup. - keyhash := hex.EncodeToString(publicKey[:]) + keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) err = db.CheckKeyHash(keyhash) // exit app if target key was modified after app start-up, e.g. if key has been deprecated if err != nil { @@ -114,7 +109,7 @@ func main() { if err != nil { msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -136,16 +131,16 @@ func main() { if err != nil { msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } - newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, Conf.RotateKey.Grpc) + newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, conf.RotateKey.Grpc) if err != nil { msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -153,7 +148,7 @@ func main() { err := errors.New("reencrypt returned empty header") msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -162,7 +157,7 @@ func main() { if err := db.StoreHeader(newHeader, fileID); err != nil { msg := fmt.Sprintf("StoreHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -171,7 +166,7 @@ func main() { if err := db.SetKeyHash(keyhash, fileID); err != nil { msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -180,7 +175,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -190,25 +185,25 @@ func main() { if err != nil { msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } reVerifyMsg, _ := json.Marshal(&reVerify) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", Conf.Broker.SchemasPath), reVerifyMsg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), reVerifyMsg) if err != nil { msg := "Validation of outgoing re-verify message failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } - if err := mq.SendMessage(delivered.CorrelationId, Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, Conf.Broker.Exchange, msg, err.Error()) + NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } From 397e0b8cd2f7b24b30b3bc94219ee369566b2ed1 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 24 Sep 2025 12:09:21 +0200 Subject: [PATCH 104/184] remove obsolete c4gh test key --- .github/integration/scripts/make_sda_credentials.sh | 5 ----- .github/integration/sda/config.yaml | 2 -- sda/config_local.yaml | 4 ++-- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index b56dbe16a..465fd827a 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -106,11 +106,6 @@ if [ ! -f "/shared/c4gh.sec.pem" ]; then /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass fi -if [ ! -f "/shared/c4gh1.sec.pem" ]; then - echo "creating crypth4gh key" - /shared/crypt4gh generate -n /shared/c4gh1 -p c4ghpass -fi - if [ ! -f "/shared/client.sec.pem" ]; then # client key for re-encryption echo "creating client crypth4gh key" /shared/crypt4gh generate -n /shared/client -p c4ghpass diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 066c4d11b..82733287c 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -78,8 +78,6 @@ c4gh: privateKeys: - filePath: /shared/c4gh.sec.pem passphrase: "c4ghpass" - - filePath: /shared/c4gh1.sec.pem - passphrase: "c4ghpass" - filePath: /shared/rotatekey.sec.pem passphrase: "rotatekeyPass" diff --git a/sda/config_local.yaml b/sda/config_local.yaml index d2eb9f088..d4f70f3c7 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -80,8 +80,8 @@ c4gh: privateKeys: - filePath: "/tmp/shared/c4gh.sec.pem" passphrase: "c4ghpass" - - filePath: "/tmp/shared/c4gh1.sec.pem" - passphrase: "c4ghpass" + - filePath: "/tmp/shared/rotatekey.sec.pem" + passphrase: "rotatekeyPass" oidc: configuration: From c040a11b3b2d8bfacb8b4c876be6844fdc96b076 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Sun, 28 Sep 2025 11:28:29 +0200 Subject: [PATCH 105/184] add logic to exit gracefully --- sda/cmd/rotatekey/rotatekey.go | 70 +++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index b50a258d8..baefdcf58 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -12,6 +12,9 @@ import ( "encoding/json" "errors" "fmt" + "os" + "os/signal" + "syscall" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -24,24 +27,56 @@ import ( ) func main() { + var ( + mq *broker.AMQPBroker + db *database.SDAdb + ) + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + if mq != nil { + defer mq.Channel.Close() + defer mq.Connection.Close() + } + if db != nil { + defer db.Close() + } + log.Fatal(err) + } + }() + forever := make(chan bool) + conf, err := config.NewConfig("rotatekey") if err != nil { - log.Fatal(err) + panic(err) } - mq, err := broker.NewMQ(conf.Broker) + mq, err = broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + panic(err) } - db, err := database.NewSDAdb(conf.Database) + db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + panic(err) } + go func() { + <-sigc // blocks here until it receives from sigc + fmt.Println("Interrupt signal received. Shutting down.") + defer mq.Channel.Close() + defer mq.Connection.Close() + defer db.Close() + + os.Exit(0) // exit program + }() + // encode pubkey as pem and then as base64 string tmp := &bytes.Buffer{} if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *conf.RotateKey.PublicKey); err != nil { - log.Fatal(err) + panic(err) } pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) @@ -49,13 +84,9 @@ func main() { keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) err = db.CheckKeyHash(keyhash) if err != nil { - log.Fatalf("database lookup of the rotation key failed, reason: %v", err) + panic(fmt.Errorf("database lookup of the rotation key failed, reason: %v", err)) } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() - go func() { connError := mq.ConnectionWatcher() log.Error(connError) @@ -72,9 +103,22 @@ func main() { var message schema.KeyRotation go func() { + // Create a function to handle panic and exit gracefully + defer func() { + if err := recover(); err != nil { + if mq != nil { + defer mq.Channel.Close() + defer mq.Connection.Close() + } + if db != nil { + defer db.Close() + } + log.Fatal(err) + } + }() messages, err := mq.GetMessages(conf.Broker.Queue) if err != nil { - log.Fatal(err) + panic(err) } for delivered := range messages { log.Debugf("Received a message (corr-id: %s, message: %s)", @@ -96,7 +140,7 @@ func main() { err = db.CheckKeyHash(keyhash) // exit app if target key was modified after app start-up, e.g. if key has been deprecated if err != nil { - log.Fatalf("check of target key failed, reason: %v", err) + panic(fmt.Errorf("check of target key failed, reason: %v", err)) } // we unmarshal the message in the validation step so this is safe to do From d716864b60e8df60721c9e9d1ec859824b1bfb56 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Sun, 28 Sep 2025 11:40:57 +0200 Subject: [PATCH 106/184] more suggestions from review -nack messages that failed to be processed - do not export nacking function --- sda/cmd/rotatekey/rotatekey.go | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index baefdcf58..b95089293 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -81,8 +81,7 @@ func main() { pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) // Check that key is registered in the db at startup - keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) - err = db.CheckKeyHash(keyhash) + err = db.CheckKeyHash(hex.EncodeToString(conf.RotateKey.PublicKey[:])) if err != nil { panic(fmt.Errorf("database lookup of the rotation key failed, reason: %v", err)) } @@ -129,7 +128,7 @@ func main() { if err != nil { msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -137,9 +136,8 @@ func main() { // Fetch rotate key hash before starting work so that we make sure the hash state // has not changed since the application startup. keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) - err = db.CheckKeyHash(keyhash) // exit app if target key was modified after app start-up, e.g. if key has been deprecated - if err != nil { + if err = db.CheckKeyHash(keyhash); err != nil { panic(fmt.Errorf("check of target key failed, reason: %v", err)) } @@ -153,7 +151,7 @@ func main() { if err != nil { msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -175,7 +173,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -184,7 +182,7 @@ func main() { if err != nil { msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -192,7 +190,7 @@ func main() { err := errors.New("reencrypt returned empty header") msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -210,7 +208,7 @@ func main() { if err := db.SetKeyHash(keyhash, fileID); err != nil { msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -219,7 +217,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -229,7 +227,7 @@ func main() { if err != nil { msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -239,7 +237,7 @@ func main() { if err != nil { msg := "Validation of outgoing re-verify message failed" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -247,7 +245,7 @@ func main() { if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) continue } @@ -261,8 +259,8 @@ func main() { <-forever } -// Nack message without requeue. Send the message to an error queue so it can be analyzed. -func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, mqExchange, msg, reason string) { +// Nack message and send the payload to an error queue so it can be analyzed. +func nackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, mqExchange, msg, reason string) { infoErrorMessage := broker.InfoError{ Error: msg, Reason: reason, @@ -273,7 +271,7 @@ func NackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, if err := mq.SendMessage(delivered.CorrelationId, mqExchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } - if err := delivered.Ack(false); err != nil { + if err := delivered.Nack(false, false); err != nil { log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } } From ec6cbbd95ffcf3c358440a302541cce36df68ca4 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Mon, 29 Sep 2025 10:13:11 +0200 Subject: [PATCH 107/184] add test for reencrypt_utils --- .../reencrypt/reencrypt_utils_test.go | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 sda/internal/reencrypt/reencrypt_utils_test.go diff --git a/sda/internal/reencrypt/reencrypt_utils_test.go b/sda/internal/reencrypt/reencrypt_utils_test.go new file mode 100644 index 000000000..8d4edbd86 --- /dev/null +++ b/sda/internal/reencrypt/reencrypt_utils_test.go @@ -0,0 +1,62 @@ +package reencrypt + +import ( + "context" + "fmt" + "net" + "testing" + + "github.com/neicnordic/sensitive-data-archive/internal/config" + "google.golang.org/grpc" +) + +type mockServer struct { + UnimplementedReencryptServer + headerResponse []byte +} + +func (s *mockServer) ReencryptHeader(ctx context.Context, req *ReencryptRequest) (*ReencryptResponse, error) { + return &ReencryptResponse{Header: s.headerResponse}, nil +} + +func TestCallReencryptHeader(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to listen: %v", err) + } + defer lis.Close() + + srv := grpc.NewServer() + mockHeader := []byte("mocked-header") + RegisterReencryptServer(srv, &mockServer{headerResponse: mockHeader}) + + go func() { + _ = srv.Serve(lis) + }() + defer srv.GracefulStop() + + host, portStr, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("failed to split host/port: %v", err) + } + var port int + _, err = fmt.Sscanf(portStr, "%d", &port) + if err != nil { + t.Fatalf("failed to parse port: %v", err) + } + + grpcConf := config.Grpc{ + Host: host, + Port: port, + Timeout: 2, + } + oldHeader := []byte("old-header") + pubKey := "test-pubkey" + res, err := CallReencryptHeader(oldHeader, pubKey, grpcConf) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if string(res) != string(mockHeader) { + t.Errorf("expected header %q, got %q", mockHeader, res) + } +} From 83ad4d07f60750dfbafa9b38efa8ca9285bb7f9c Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 29 Sep 2025 11:20:35 +0200 Subject: [PATCH 108/184] check that fileID is uuid conformant - use simlink instead of duplicate json file --- .../tests/sda/70_rotate_key_test.sh | 5 ++-- sda/schemas/federated/rotate-key.json | 1 + sda/schemas/isolated/rotate-key.json | 30 +------------------ 3 files changed, 5 insertions(+), 31 deletions(-) mode change 100644 => 120000 sda/schemas/isolated/rotate-key.json diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 51de6337d..297e3ff39 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -226,7 +226,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ checkConsumers rotatekey 0 ## test app attempts to start with a configured rotation key that is deprecated -echo "test app fails to start with a configured rotation key that is invalid" +echo "test that app fails to start with a configured rotation key that is invalid" sleep 2 # app will keep failing until we restore tha target key as active @@ -238,11 +238,12 @@ psql -U postgres -h postgres -d sda -At -c "UPDATE sda.encryption_keys SET depre checkConsumers rotatekey 1 ## test bad message -test "test bad mq message" +echo "test bad mq message" rotatekey_payload_bad=$( jq -r -c -n \ --arg type "key_rotation" \ + --arg file_id "0f38b6z-9868-446f-91ab-6a83832a3f0a" \ '$ARGS.named|@base64' ) diff --git a/sda/schemas/federated/rotate-key.json b/sda/schemas/federated/rotate-key.json index cec407ca9..8fd5f0904 100644 --- a/sda/schemas/federated/rotate-key.json +++ b/sda/schemas/federated/rotate-key.json @@ -21,6 +21,7 @@ "type": "string", "title": "The unique file identifier", "description": "The unique file identifier", + "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$", "examples": [ "420420cc43-e060-4583-a891-9f8170ee66c8" ] diff --git a/sda/schemas/isolated/rotate-key.json b/sda/schemas/isolated/rotate-key.json deleted file mode 100644 index 6d6bc39d5..000000000 --- a/sda/schemas/isolated/rotate-key.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "title": "JSON schema for SDA key rotation message interface", - "$id": "https://github.com/neicnordic/sensitive-data-archive/tree/master/sda/schemas/isolated/rotate-key.json", - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "required": [ - "type", - "file_id" - ], - "additionalProperties": true, - "properties": { - "type": { - "$id": "#/properties/type", - "type": "string", - "title": "The message type", - "description": "The message type", - "const": "key_rotation" - }, - "file_id": { - "$id": "#/properties/file_id", - "type": "string", - "title": "The unique file identifier", - "description": "The unique file identifier", - "examples": [ - "420420cc43-e060-4583-a891-9f8170ee66c8" - ] - } - } -} diff --git a/sda/schemas/isolated/rotate-key.json b/sda/schemas/isolated/rotate-key.json new file mode 120000 index 000000000..09e7a2493 --- /dev/null +++ b/sda/schemas/isolated/rotate-key.json @@ -0,0 +1 @@ +../federated/rotate-key.json \ No newline at end of file From d1253e512cf3dd01fb50138a7ac0ee9c6d4716ea Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 30 Sep 2025 11:51:58 +0200 Subject: [PATCH 109/184] add RotateHeaderKey function - single query updates file header and keyhash --- sda/internal/database/db_functions.go | 33 +++++++++++++++ sda/internal/database/db_functions_test.go | 49 ++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 34942cf4d..732979213 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -200,6 +200,39 @@ func (dbs *SDAdb) storeHeader(header []byte, id string) error { return nil } +// RotateHeader stores the file header in the database +func (dbs *SDAdb) RotateHeaderKey(header []byte, keyHash, fileID string) error { + var ( + err error + count int + ) + + for count == 0 || (err != nil && count < RetryTimes) { + err = dbs.rotateHeaderKey(header, keyHash, fileID) + count++ + } + + return err +} + +func (dbs *SDAdb) rotateHeaderKey(header []byte, keyHash, fileID string) error { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const query = "UPDATE sda.files SET header = $1, key_hash = $2 WHERE id = $3;" + + result, err := db.Exec(query, hex.EncodeToString(header), keyHash, fileID) + if err != nil { + return err + } + if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { + return errors.New("something went wrong with the query zero rows were changed") + } + log.Debugf("Successfully set header and key hash for file %s", fileID) + + return nil +} + // SetArchived marks the file as 'ARCHIVED' func (dbs *SDAdb) SetArchived(file FileInfo, fileID string) error { var err error diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 54e989518..afc2e11ca 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -2,6 +2,7 @@ package database import ( "crypto/sha256" + "encoding/hex" "fmt" "regexp" "time" @@ -103,6 +104,54 @@ func (suite *DatabaseTests) TestStoreHeader() { db.Close() } +func (suite *DatabaseTests) TestRotateHeaderKey() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // Register a new key and a new file + fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + err = db.addKeyHash("someKeyHash", "this is a test key") + assert.NoError(suite.T(), err, "failed to register key in database") + err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) + assert.NoError(suite.T(), err, "failed to store file header") + + // test happy path + newKeyHex := `6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507` + err = db.addKeyHash(newKeyHex, "new key") + assert.NoError(suite.T(), err, "failed to register key in database") + newHHeader := []byte{1, 2, 3} + + err = db.RotateHeaderKey(newHHeader, newKeyHex, fileID) + assert.NoError(suite.T(), err) + + // Verify that the key+header were updated + var dbHeaderString, dbKeyHash string + err = db.DB.QueryRow("SELECT header, key_hash FROM sda.files WHERE id=$1", fileID).Scan(&dbHeaderString, &dbKeyHash) + assert.NoError(suite.T(), err) + dbHeader, err := hex.DecodeString(dbHeaderString) + assert.NoError(suite.T(), err, "hex decoding of rotated header failed") + assert.Equal(suite.T(), newHHeader, dbHeader) + assert.Equal(suite.T(), newKeyHex, dbKeyHash) + + // case of non registered keyhash + err = db.RotateHeaderKey([]byte{2, 4, 6, 8}, "unknownKeyHash", fileID) + assert.ErrorContains(suite.T(), err, "violates foreign key constraint") + // check that no column was updated + err = db.DB.QueryRow("SELECT header, key_hash FROM sda.files WHERE id=$1", fileID).Scan(&dbHeaderString, &dbKeyHash) + assert.NoError(suite.T(), err) + dbHeader, err = hex.DecodeString(dbHeaderString) + assert.NoError(suite.T(), err, "hex decoding of rotated header failed") + assert.Equal(suite.T(), newHHeader, dbHeader) + assert.Equal(suite.T(), newKeyHex, dbKeyHash) + + // case of non existing entry + err = db.RotateHeaderKey([]byte{15, 45, 20, 40, 48}, "keyHex", "00000000-0000-0000-0000-000000000000") + assert.EqualError(suite.T(), err, "something went wrong with the query zero rows were changed") + + db.Close() +} + func (suite *DatabaseTests) TestSetArchived() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) From 29b271e64adc92d1bfe228d933dae375d5513e36 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 30 Sep 2025 11:54:46 +0200 Subject: [PATCH 110/184] update header and keyhash together --- sda/cmd/rotatekey/rotatekey.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index b95089293..b7ecff20e 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -195,18 +195,9 @@ func main() { continue } - // Rotate header in database - if err := db.StoreHeader(newHeader, fileID); err != nil { - msg := fmt.Sprintf("StoreHeader failed for file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - NackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) - - continue - } - - // Rotate keyhash - if err := db.SetKeyHash(keyhash, fileID); err != nil { - msg := fmt.Sprintf("SetKeyHash failed for file-id: %s", fileID) + // Rotate header and keyhash in database + if err := db.RotateHeaderKey(newHeader, keyhash, fileID); err != nil { + msg := fmt.Sprintf("RotateHeaderKey failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) From 86a297819b0d6cc1346b336abe58546a84545077 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 1 Oct 2025 22:33:34 +0200 Subject: [PATCH 111/184] refactor rotatekey to ease unittesting --- sda/cmd/rotatekey/rotatekey.go | 237 +++++++++++++++++---------------- 1 file changed, 123 insertions(+), 114 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index b7ecff20e..e89887342 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -26,23 +26,29 @@ import ( log "github.com/sirupsen/logrus" ) +type RotateKey struct { + Conf *config.Config + MQ *broker.AMQPBroker + DB *database.SDAdb + PubKeyEncoded string +} + func main() { - var ( - mq *broker.AMQPBroker - db *database.SDAdb - ) + app := RotateKey{} + var err error + sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // Create a function to handle panic and exit gracefully defer func() { if err := recover(); err != nil { - if mq != nil { - defer mq.Channel.Close() - defer mq.Connection.Close() + if app.MQ != nil { + defer app.MQ.Channel.Close() + defer app.MQ.Connection.Close() } - if db != nil { - defer db.Close() + if app.DB != nil { + defer app.DB.Close() } log.Fatal(err) } @@ -50,15 +56,15 @@ func main() { forever := make(chan bool) - conf, err := config.NewConfig("rotatekey") + app.Conf, err = config.NewConfig("rotatekey") if err != nil { panic(err) } - mq, err = broker.NewMQ(conf.Broker) + app.MQ, err = broker.NewMQ(app.Conf.Broker) if err != nil { panic(err) } - db, err = database.NewSDAdb(conf.Database) + app.DB, err = database.NewSDAdb(app.Conf.Database) if err != nil { panic(err) } @@ -66,34 +72,34 @@ func main() { go func() { <-sigc // blocks here until it receives from sigc fmt.Println("Interrupt signal received. Shutting down.") - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() + defer app.MQ.Channel.Close() + defer app.MQ.Connection.Close() + defer app.DB.Close() os.Exit(0) // exit program }() // encode pubkey as pem and then as base64 string tmp := &bytes.Buffer{} - if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *conf.RotateKey.PublicKey); err != nil { + if err := keys.WriteCrypt4GHX25519PublicKey(tmp, *app.Conf.RotateKey.PublicKey); err != nil { panic(err) } - pubKeyEncoded := base64.StdEncoding.EncodeToString(tmp.Bytes()) + app.PubKeyEncoded = base64.StdEncoding.EncodeToString(tmp.Bytes()) // Check that key is registered in the db at startup - err = db.CheckKeyHash(hex.EncodeToString(conf.RotateKey.PublicKey[:])) + err = app.DB.CheckKeyHash(hex.EncodeToString(app.Conf.RotateKey.PublicKey[:])) if err != nil { panic(fmt.Errorf("database lookup of the rotation key failed, reason: %v", err)) } go func() { - connError := mq.ConnectionWatcher() + connError := app.MQ.ConnectionWatcher() log.Error(connError) forever <- false }() go func() { - connError := mq.ChannelWatcher() + connError := app.MQ.ChannelWatcher() log.Error(connError) forever <- false }() @@ -105,17 +111,17 @@ func main() { // Create a function to handle panic and exit gracefully defer func() { if err := recover(); err != nil { - if mq != nil { - defer mq.Channel.Close() - defer mq.Connection.Close() + if app.MQ != nil { + defer app.MQ.Channel.Close() + defer app.MQ.Connection.Close() } - if db != nil { - defer db.Close() + if app.DB != nil { + defer app.DB.Close() } log.Fatal(err) } }() - messages, err := mq.GetMessages(conf.Broker.Queue) + messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) if err != nil { panic(err) } @@ -124,134 +130,137 @@ func main() { delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", conf.Broker.SchemasPath), delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + app.nackAndSendToErrorQueue(delivered, msg, err.Error()) continue } // Fetch rotate key hash before starting work so that we make sure the hash state // has not changed since the application startup. - keyhash := hex.EncodeToString(conf.RotateKey.PublicKey[:]) + keyhash := hex.EncodeToString(app.Conf.RotateKey.PublicKey[:]) // exit app if target key was modified after app start-up, e.g. if key has been deprecated - if err = db.CheckKeyHash(keyhash); err != nil { + if err = app.DB.CheckKeyHash(keyhash); err != nil { panic(fmt.Errorf("check of target key failed, reason: %v", err)) } // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - fileID := message.FileID - - // Get current keyhash for the file, send to error queue if this fails - oldKeyHash, err := db.GetKeyHash(fileID) - if err != nil { - msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) - - continue - } + ackNack, msg, err := app.rotateHeader(delivered.CorrelationId, message.FileID) - // Check that the file is not already encrypted with the target key - if oldKeyHash == keyhash { - log.Infof("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) + switch ackNack { + case "ack": if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack following already encrypted with key message") + log.Errorf("failed to ack message, reason: %v", err) + } + case "nack": + app.nackAndSendToErrorQueue(delivered, msg, err.Error()) + default: + // will catch `reject`s, failures that should not be requeued. + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message, reason: %v", err) } - - continue } + } + }() - // reencrypt header - log.Debugf("rotating c4gh key for file with file-id: %s", fileID) + <-forever +} - header, err := db.GetHeader(fileID) - if err != nil { - msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) +func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg string, err error) { + // Get current keyhash for the file, send to error queue if this fails + oldKeyHash, err := app.DB.GetKeyHash(fileID) + if err != nil { + msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - continue - } + return "nack", msg, err + } - newHeader, err := reencrypt.CallReencryptHeader(header, pubKeyEncoded, conf.RotateKey.Grpc) - if err != nil { - msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + // Check that the file is not already encrypted with the target key + keyhash := hex.EncodeToString(app.Conf.RotateKey.PublicKey[:]) + if oldKeyHash == keyhash { + log.Infof("the file with file-id: %s is already encrypted with the given rotation c4gh key", fileID) - continue - } - if newHeader == nil { - err := errors.New("reencrypt returned empty header") - msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + return "ack", "", nil + } - continue - } + // reencrypt header + log.Debugf("rotating c4gh key for file with file-id: %s", fileID) - // Rotate header and keyhash in database - if err := db.RotateHeaderKey(newHeader, keyhash, fileID); err != nil { - msg := fmt.Sprintf("RotateHeaderKey failed for file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + header, err := app.DB.GetHeader(fileID) + if err != nil { + msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - continue - } + return "nack", msg, err + } - aID, err := db.GetAccessionID(fileID) - if err != nil { - msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + newHeader, err := reencrypt.CallReencryptHeader(header, app.PubKeyEncoded, app.Conf.RotateKey.Grpc) + if err != nil { + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - continue - } + return "nack", msg, err + } + if newHeader == nil { + err := errors.New("reencrypt returned empty header") + msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - // Send re-verify message - reVerify, err := db.GetReVerificationData(aID) - if err != nil { - msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + return "nack", msg, err + } - continue - } + // Rotate header and keyhash in database + if err := app.DB.RotateHeaderKey(newHeader, keyhash, fileID); err != nil { + msg := fmt.Sprintf("RotateHeaderKey failed for file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - reVerifyMsg, _ := json.Marshal(&reVerify) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), reVerifyMsg) - if err != nil { - msg := "Validation of outgoing re-verify message failed" - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + return "nack", msg, err + } - continue - } + aID, err := app.DB.GetAccessionID(fileID) + if err != nil { + msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { - msg := "failed to publish message" - log.Errorf("%s, reason: %v", msg, err) - nackAndSendToErrorQueue(mq, delivered, conf.Broker.Exchange, msg, err.Error()) + return "nack", msg, err + } - continue - } + // Send re-verify message + reVerify, err := app.DB.GetReVerificationData(aID) + if err != nil { + msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) + log.Errorf("%s, reason: %v", msg, err) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - } - }() + return "nack", msg, err + } - <-forever + reVerifyMsg, _ := json.Marshal(&reVerify) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), reVerifyMsg) + if err != nil { + msg := "Validation of outgoing re-verify message failed" + log.Errorf("%s, reason: %v", msg, err) + + return "nack", msg, err + } + + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + msg := "failed to publish message" + log.Errorf("%s, reason: %v", msg, err) + + return "nack", msg, err + } + + return "ack", "", nil } // Nack message and send the payload to an error queue so it can be analyzed. -func nackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, mqExchange, msg, reason string) { +func (app *RotateKey) nackAndSendToErrorQueue(delivered amqp091.Delivery, msg, reason string) { infoErrorMessage := broker.InfoError{ Error: msg, Reason: reason, @@ -259,7 +268,7 @@ func nackAndSendToErrorQueue(mq *broker.AMQPBroker, delivered amqp091.Delivery, } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, mqExchange, "error", body); err != nil { + if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Nack(false, false); err != nil { From 5b7c625f0a73cfd898bf387c96787a545fd94ff6 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 2 Oct 2025 15:33:19 +0200 Subject: [PATCH 112/184] refactor message handling --- sda/cmd/rotatekey/rotatekey.go | 69 +++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index e89887342..074b6bb6d 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -22,7 +22,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) @@ -134,7 +133,19 @@ func main() { if err != nil { msg := "validation of incoming message (rotate-key) failed" log.Errorf("%s, reason: %v", msg, err) - app.nackAndSendToErrorQueue(delivered, msg, err.Error()) + // Ack message and send the payload to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: msg, + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + body, _ := json.Marshal(infoErrorMessage) + if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } continue } @@ -157,8 +168,23 @@ func main() { if err := delivered.Ack(false); err != nil { log.Errorf("failed to ack message, reason: %v", err) } - case "nack": - app.nackAndSendToErrorQueue(delivered, msg, err.Error()) + case "ackSendToError": + infoErrorMessage := broker.InfoError{ + Error: msg, + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + body, _ := json.Marshal(infoErrorMessage) + if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + case "nackRequeue": + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) + } default: // will catch `reject`s, failures that should not be requeued. if err := delivered.Reject(false); err != nil { @@ -178,7 +204,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } // Check that the file is not already encrypted with the target key @@ -197,7 +223,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } newHeader, err := reencrypt.CallReencryptHeader(header, app.PubKeyEncoded, app.Conf.RotateKey.Grpc) @@ -205,14 +231,14 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } if newHeader == nil { err := errors.New("reencrypt returned empty header") msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } // Rotate header and keyhash in database @@ -220,7 +246,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("RotateHeaderKey failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } aID, err := app.DB.GetAccessionID(fileID) @@ -228,7 +254,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } // Send re-verify message @@ -237,7 +263,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } reVerifyMsg, _ := json.Marshal(&reVerify) @@ -246,32 +272,15 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := "Validation of outgoing re-verify message failed" log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - return "nack", msg, err + return "ackSendToError", msg, err } return "ack", "", nil } - -// Nack message and send the payload to an error queue so it can be analyzed. -func (app *RotateKey) nackAndSendToErrorQueue(delivered amqp091.Delivery, msg, reason string) { - infoErrorMessage := broker.InfoError{ - Error: msg, - Reason: reason, - OriginalMessage: string(delivered.Body), - } - body, _ := json.Marshal(infoErrorMessage) - - if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } -} From dd80f8c882500d1be5259557cacfaaa6e1631aff Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 2 Oct 2025 17:59:16 +0200 Subject: [PATCH 113/184] requeue message if db error is recoverable and don't otherwise --- .../tests/sda/70_rotate_key_test.sh | 32 +++++++++++++++++++ sda/cmd/rotatekey/rotatekey.go | 19 ++++++++--- sda/internal/database/db_functions.go | 2 +- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 297e3ff39..8a9da359c 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -264,4 +264,36 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ checkErrors "validation of incoming message (rotate-key) failed" +# update errorStream +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +## test non-existent fileID +echo "test non-existent fileID" + +rotatekey_payload_bad=$( + jq -r -c -n \ + --arg type "key_rotation" \ + --arg file_id "d3fc4148-6918-479c-914d-ad669041c816" \ + '$ARGS.named|@base64' +) + +rotatekey_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "rotatekey" \ + --arg payload_encoding base64 \ + --arg payload "$rotatekey_payload_bad" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$rotatekey_body" | jq + +checkErrors "failed to get keyhash for file" + +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + printf "\033[32mRotate key integration tests completed successfully\033[0m\n" diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 074b6bb6d..6b5ff1f12 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" "github.com/neicnordic/crypt4gh/keys" @@ -204,7 +205,12 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "ackSendToError", msg, err + switch { + case strings.Contains(err.Error(), "sql: no rows in result set"): + return "ackSendToError", msg, err + default: + return "nackRequeue", msg, err + } } // Check that the file is not already encrypted with the target key @@ -223,7 +229,12 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("GetHeader failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "ackSendToError", msg, err + switch { + case strings.Contains(err.Error(), "sql: no rows in result set"): + return "ackSendToError", msg, err + default: + return "nackRequeue", msg, err + } } newHeader, err := reencrypt.CallReencryptHeader(header, app.PubKeyEncoded, app.Conf.RotateKey.Grpc) @@ -246,7 +257,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := fmt.Sprintf("RotateHeaderKey failed for file-id: %s", fileID) log.Errorf("%s, reason: %v", msg, err) - return "ackSendToError", msg, err + return "nackRequeue", msg, err } aID, err := app.DB.GetAccessionID(fileID) @@ -279,7 +290,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - return "ackSendToError", msg, err + return "nackRequeue", msg, err } return "ack", "", nil diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 732979213..b49003a3d 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1010,7 +1010,7 @@ func (dbs *SDAdb) GetKeyHash(fileID string) (string, error) { // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { keyHash, err = dbs.getKeyHash(fileID) - if err == nil { + if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { break } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) From f3b629145b321f165b1b7dba87bd20d5cf00798a Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 2 Oct 2025 20:22:23 +0200 Subject: [PATCH 114/184] add func to get reverification data from fileID --- sda/internal/database/db_functions.go | 24 +++++++++++++++ sda/internal/database/db_functions_test.go | 36 ++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index b49003a3d..2bd729bb2 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1252,6 +1252,30 @@ func (dbs *SDAdb) GetReVerificationData(accessionID string) (schema.IngestionVer return reVerify, nil } +func (dbs *SDAdb) GetReVerificationDataFromFileID(fileID string) (schema.IngestionVerification, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + reVerify := schema.IngestionVerification{ReVerify: true, FileID: fileID} + + const query = "SELECT archive_file_path,submission_file_path,submission_user FROM sda.files where id = $1;" + err := db.QueryRow(query, fileID).Scan(&reVerify.ArchivePath, &reVerify.FilePath, &reVerify.User) + if err != nil { + return schema.IngestionVerification{}, err + } + + var checksum schema.Checksums + const archiveChecksum = "SELECT type,checksum from sda.checksums WHERE file_id = $1 AND source = 'ARCHIVED';" + if err := db.QueryRow(archiveChecksum, reVerify.FileID).Scan(&checksum.Type, &checksum.Value); err != nil { + log.Errorln(err.Error()) + + return schema.IngestionVerification{}, err + } + checksum.Type = strings.ToLower(checksum.Type) + reVerify.EncryptedChecksums = append(reVerify.EncryptedChecksums, checksum) + + return reVerify, nil +} + func (dbs *SDAdb) GetDecryptedChecksum(id string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index afc2e11ca..23c563e79 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1360,6 +1360,42 @@ func (suite *DatabaseTests) TestGetReVerificationData() { db.Close() } +func (suite *DatabaseTests) TestGetReVerificationDataFromFileID() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + fileID, err := db.RegisterFile("/testuser/TestGetReVerificationData.c4gh", "testuser") + if err != nil { + suite.FailNow("failed to register file in database") + } + + encSha := sha256.New() + _, err = encSha.Write([]byte("Checksum")) + if err != nil { + suite.FailNow("failed to generate checksum") + } + + decSha := sha256.New() + _, err = decSha.Write([]byte("DecryptedChecksum")) + if err != nil { + suite.FailNow("failed to generate checksum") + } + + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} + if err = db.SetArchived(fileInfo, fileID); err != nil { + suite.FailNow("failed to archive file") + } + if err = db.SetVerified(fileInfo, fileID); err != nil { + suite.FailNow("failed to mark file as verified") + } + + data, err := db.GetReVerificationDataFromFileID(fileID) + assert.NoError(suite.T(), err, "failed to get verification data from fileID") + assert.Equal(suite.T(), "/archive/TestGetReVerificationData.c4gh", data.ArchivePath) + + db.Close() +} + func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) From 79ae6f42426d87027a44d1c436e8810cdad28cfe Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 2 Oct 2025 20:24:31 +0200 Subject: [PATCH 115/184] remove any dependency on accessionID as input - retrieve reverify data from fileID - rework integration test --- .../tests/sda/70_rotate_key_test.sh | 43 +++++++------------ sda/cmd/rotatekey/rotatekey.go | 10 +---- 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 8a9da359c..c952d4b6f 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -7,19 +7,6 @@ fi cd shared || true -checkStatus () { - RETRY_TIMES=0 - until [ "$(curl -s -k -H "Authorization: Bearer $token" -X GET http://api:8080/users/test@dummy.org/files | jq | grep -c "$1")" -eq "$2" ]; do - echo "waiting for files to become $1" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for files to become $1" - exit 1 - fi - sleep 2 - done -} - checkErrors() { RETRY_TIMES=0 until [ $(("$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')"-"$errorStreamSize")) -eq 1 ]; do @@ -88,15 +75,18 @@ if [ "$response" -ne 1 ]; then exit 1 fi -## ingest and map files to dataset +## ingest file curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/ingest -checkStatus verified 1 - -curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_id": "ROTATE-KEY-01", "filepath": "dataset_rotatekey/testfile1.c4gh", "user": "test@dummy.org"}' http://api:8080/file/accession -checkStatus ready 1 - -curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["ROTATE-KEY-01"], "dataset_id": "KEY-ROTATION-TEST-0001", "user": "test@dummy.org"}' http://api:8080/dataset/create -checkStatus ready 0 +RETRY_TIMES=0 +until [ "$(curl -s -k -H "Authorization: Bearer $token" -X GET http://api:8080/users/test@dummy.org/files | jq | grep -c "verified")" -eq 1 ]; do + echo "waiting for files to become verified" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to become verified" + exit 1 + fi + sleep 2 +done errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') @@ -107,7 +97,7 @@ corrID=$( -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id ) -fileID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where stable_id='ROTATE-KEY-01';") +fileID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_rotatekey/testfile1.c4gh';") properties=$( jq -c -n \ @@ -142,7 +132,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ # check DB for updated key hash in sda.files rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") -if [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where stable_id like 'ROTATE-KEY-0%';" | grep -c "$rotatekeyHash")" -ne 1 ]; +if [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID';" | grep -c "$rotatekeyHash")" -ne 1 ]; then echo "failed to update the key hash of files" exit 1 @@ -162,6 +152,7 @@ until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/archived/ | done # check that no other erros occured +sleep 5 if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')" -ne "$errorStreamSize" ]; then echo "something went wrong with the key rotation" exit 1 @@ -170,10 +161,10 @@ fi ## download file with rotated key, concatenate header and archive body, decrypt and check # get rotated header -psql -U postgres -h postgres -d sda -At -c "select header from sda.files where stable_id='ROTATE-KEY-01';" | xxd -r -p > testfile1_rotated.c4gh +psql -U postgres -h postgres -d sda -At -c "select header from sda.files where id='$fileID';" | xxd -r -p > testfile1_rotated.c4gh # get archive file -archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where stable_id='ROTATE-KEY-01';") +archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where id='$fileID';") s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force # concatenate and decrypt @@ -294,6 +285,4 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ checkErrors "failed to get keyhash for file" -errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') - printf "\033[32mRotate key integration tests completed successfully\033[0m\n" diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 6b5ff1f12..e634d47b4 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -260,16 +260,8 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s return "nackRequeue", msg, err } - aID, err := app.DB.GetAccessionID(fileID) - if err != nil { - msg := fmt.Sprintf("GetAccessionID failed for file-id: %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - - return "ackSendToError", msg, err - } - // Send re-verify message - reVerify, err := app.DB.GetReVerificationData(aID) + reVerify, err := app.DB.GetReVerificationDataFromFileID(fileID) if err != nil { msg := fmt.Sprintf("GetReVerificationData failed for file-id %s", fileID) log.Errorf("%s, reason: %v", msg, err) From 8b23ac886378c0147c7c1ceaf634d873c52f2be9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 12:55:57 +0200 Subject: [PATCH 116/184] [Test CallReEncryptHeader] ensure correct error response --- sda/cmd/reencrypt/reencrypt_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 897fa450d..90ab3fd4b 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -481,6 +481,6 @@ func (ts *ReEncryptTests) TestCallReencryptHeader_BadInput() { } res, err := re.CallReencryptHeader(ts.FileHeader, "somekey", grpcConf) - assert.Error(ts.T(), err) + assert.ErrorContains(ts.T(), err, "illegal base64 data") assert.Nil(ts.T(), res) } From a0a5468be13b7040c7744ee5e757fba518a986ad Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 13:00:39 +0200 Subject: [PATCH 117/184] [reencrypt] ensure an error message is always returned on error --- sda/cmd/reencrypt/reencrypt.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index 0f954c3fb..e3ade4da8 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -97,7 +97,7 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r } } - return nil, status.Error(400, err.Error()) + return nil, status.Error(400, "header reencryption failed, no matching key available") } // Check implements the healthgrpc.HealthServer Check method for the proxy grpc Health server. From ce42231f2b924ba703fac3cf31fd1c90ee1bbfa3 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 13:01:38 +0200 Subject: [PATCH 118/184] [reencrypt] add test for failed reencryption due to no matching key --- sda/cmd/reencrypt/reencrypt_test.go | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 90ab3fd4b..99a39d1e3 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -484,3 +484,40 @@ func (ts *ReEncryptTests) TestCallReencryptHeader_BadInput() { assert.ErrorContains(ts.T(), err, "illegal base64 data") assert.Nil(ts.T(), res) } + +func (ts *ReEncryptTests) TestReencryptHeader_NoMatchingKey() { + lis, err := net.Listen("tcp", "localhost:50065") + if err != nil { + ts.T().FailNow() + } + + var keyList []*[32]byte + _, testKey, err := keys.GenerateKeyPair() + if err != nil { + ts.T().FailNow() + } + keyList = append(keyList, (&testKey)) + + go func() { + var opts []grpc.ServerOption + s := grpc.NewServer(opts...) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: keyList}) + _ = s.Serve(lis) + }() + + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient("localhost:50065", opts...) + if err != nil { + ts.T().FailNow() + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + c := re.NewReencryptClient(conn) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString}) + assert.Contains(ts.T(), err.Error(), "reencryption failed, no matching key available") + assert.Nil(ts.T(), res) +} From 3e9ee5007d596f13e8eae4596e1f2b52e17897b1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 13:14:32 +0200 Subject: [PATCH 119/184] [rotateKey] remove check for non exiting nil value --- sda/cmd/rotatekey/rotatekey.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index e634d47b4..7122dc678 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -10,7 +10,6 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "errors" "fmt" "os" "os/signal" @@ -244,13 +243,6 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s return "ackSendToError", msg, err } - if newHeader == nil { - err := errors.New("reencrypt returned empty header") - msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID) - log.Errorf("%s, reason: %v", msg, err) - - return "ackSendToError", msg, err - } // Rotate header and keyhash in database if err := app.DB.RotateHeaderKey(newHeader, keyhash, fileID); err != nil { From 93609c3eb1e3b96134bb615f124678ea1f67b0cd Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 13:18:54 +0200 Subject: [PATCH 120/184] [rotateKey] failure to trigger re-validation needs to go to error Otherwise we can not ensure file integrity after header key rotation. --- sda/cmd/rotatekey/rotatekey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index 7122dc678..b9284510e 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -274,7 +274,7 @@ func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg s msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) - return "nackRequeue", msg, err + return "ackSendToError", msg, err } return "ack", "", nil From 636698cbe9b7d7d06b049b70f59494c2820e5080 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 10 Oct 2025 13:23:18 +0200 Subject: [PATCH 121/184] [rotateKey] rename rotateHeader to reEncryptHeader --- sda/cmd/rotatekey/rotatekey.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index b9284510e..b1734e0aa 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -161,7 +161,7 @@ func main() { // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - ackNack, msg, err := app.rotateHeader(delivered.CorrelationId, message.FileID) + ackNack, msg, err := app.reEncryptHeader(delivered.CorrelationId, message.FileID) switch ackNack { case "ack": @@ -197,7 +197,7 @@ func main() { <-forever } -func (app *RotateKey) rotateHeader(correlationID, fileID string) (ackNack, msg string, err error) { +func (app *RotateKey) reEncryptHeader(correlationID, fileID string) (ackNack, msg string, err error) { // Get current keyhash for the file, send to error queue if this fails oldKeyHash, err := app.DB.GetKeyHash(fileID) if err != nil { From f93b4aa63bd8b0b8f6599c5b132e8d2faeeb5da8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 13 Oct 2025 14:13:05 +0200 Subject: [PATCH 122/184] [rotate key] add basic test suite. --- sda/cmd/rotatekey/rotatekey_test.go | 327 ++++++++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 sda/cmd/rotatekey/rotatekey_test.go diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go new file mode 100644 index 000000000..014fc5e05 --- /dev/null +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -0,0 +1,327 @@ +package main + +import ( + "context" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "net" + "net/http" + "os" + "path" + "runtime" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" + re "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" + "github.com/ory/dockertest" + "github.com/ory/dockertest/docker" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "google.golang.org/grpc" + "google.golang.org/grpc/reflection" +) + +var dbPort, mqPort int + +func TestMain(m *testing.M) { + if _, err := os.Stat("/.dockerenv"); err == nil { + m.Run() + } + _, b, _, _ := runtime.Caller(0) + rootDir := path.Join(path.Dir(b), "../../../") + + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + // uses pool to try to connect to Docker + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + postgres, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "15.4-alpine3.17", + Env: []string{ + "POSTGRES_PASSWORD=rootpasswd", + "POSTGRES_DB=sda", + }, + Mounts: []string{ + fmt.Sprintf("%s/postgresql/initdb.d:/docker-entrypoint-initdb.d", rootDir), + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + dbHostAndPort := postgres.GetHostPort("5432/tcp") + dbPort, _ = strconv.Atoi(postgres.GetPort("5432/tcp")) + databaseURL := fmt.Sprintf("postgres://postgres:rootpasswd@%s/sda?sslmode=disable", dbHostAndPort) + + pool.MaxWait = 120 * time.Second + if err = pool.Retry(func() error { + db, err := sql.Open("postgres", databaseURL) + if err != nil { + log.Println(err) + + return err + } + + query := "SELECT MAX(version) FROM sda.dbschema_version;" + var dbVersion int + + return db.QueryRow(query).Scan(&dbVersion) + }); err != nil { + log.Fatalf("Could not connect to postgres: %s", err) + } + + // pulls an image, creates a container based on it and runs it + rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "ghcr.io/neicnordic/sensitive-data-archive", + Tag: "v3.0.0-rabbitmq", + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not start resource: %s", err) + } + + mqPort, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp")) + brokerAPI := rabbitmq.GetHostPort("15672/tcp") + + client := http.Client{Timeout: 30 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+brokerAPI+"/api/queues/sda/", http.NoBody) + if err != nil { + log.Fatal(err) + } + req.SetBasicAuth("guest", "guest") + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + return err + } + res.Body.Close() + + return nil + }); err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not connect to rabbitmq: %s", err) + } + + log.Println("starting tests") + code := m.Run() + + log.Println("tests completed") + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + + os.Exit(code) +} + +type TestSuite struct { + suite.Suite + app RotateKey + corrID string + fileID string + privateKeyList []*[32]byte +} +type server struct { + re.UnimplementedReencryptServer + c4ghPrivateKeyList []*[32]byte +} + +func TestRotateKeyTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (ts *TestSuite) SetupSuite() { + ts.app.Conf = &config.Config{} + ts.app.Conf.Broker.SchemasPath = "../../schemas/isolated" + ts.corrID = uuid.New().String() + var err error + ts.app.DB, err = database.NewSDAdb(database.DBConf{ + Host: "localhost", + Port: dbPort, + User: "postgres", + Password: "rootpasswd", + Database: "sda", + SslMode: "disable", + }) + if err != nil { + ts.FailNow("Failed to create DB connection") + } + + ts.app.MQ, err = broker.NewMQ(broker.MQConf{ + Host: "localhost", + Port: mqPort, + User: "guest", + Password: "guest", + Exchange: "sda", + Vhost: "/sda", + }) + if err != nil { + ts.T().Log(err.Error()) + ts.FailNow("Failed to create MQ connection") + } + + publicKey, _, err := keys.GenerateKeyPair() + if err != nil { + ts.FailNow("Failed to create new c4gh keypair") + } + + for i, kh := range []string{"79f2f4dd9cd9435743d5e8ef3d0da55d64437055e89cfa5531395abf8857bd63", hex.EncodeToString(publicKey[:])} { + if err := ts.app.DB.AddKeyHash(kh, fmt.Sprintf("key num: %d", i)); err != nil { + ts.FailNow("failed to register a public key") + } + } + + ts.app.Conf.RotateKey.PublicKey = &publicKey + + ts.fileID, err = ts.app.DB.RegisterFile("rotate-key-test/data.c4gh", "tester_example.org") + if err != nil { + ts.FailNow("Failed to register file in DB") + } + for _, status := range []string{"uploaded", "archived", "verified"} { + if err = ts.app.DB.UpdateFileEventLog(ts.fileID, status, ts.corrID, "tester_example.org", "{}", "{}"); err != nil { + ts.FailNow("Failed to set status of file in DB") + } + } + if err := ts.app.DB.SetKeyHash("79f2f4dd9cd9435743d5e8ef3d0da55d64437055e89cfa5531395abf8857bd63", ts.fileID); err != nil { + ts.FailNow("Failed to set key hash of file in DB") + } + if err := ts.app.DB.StoreHeader([]byte("637279707434676801000000010000006c000000000000004f6ae97503ac19b6316cb3330ea4e55e0fa98ed7342afc79deec64606aa33a587e78743695f3be5d5b9d0f386c2b66aefb06de07c506eccec4910455d75f54ce6324b98b4dd35dcc6c0684bbf8a05fb5c2976f540dbbbc95646c2e55ec52c5833115e5659"), ts.fileID); err != nil { + ts.FailNow("Failed to store header of file in DB") + } + + fileInfo := database.FileInfo{ + ArchiveChecksum: "239729e2f471a02f8b43374fa58ea2d3a85ec93874b58696030b4af804c32f36", + DecryptedChecksum: "9aa63cfe45c560c8f16dde4b002a3fe38afa69801df6a6e266b757ab6aace2d8", + DecryptedSize: 34, + Path: ts.fileID, + Size: 59, + } + if err := ts.app.DB.SetVerified(fileInfo, ts.fileID); err != nil { + ts.FailNow("Failed to store header of file in DB") + } + + lis, err := net.Listen("tcp", "localhost:") + if err != nil { + log.Errorf("failed to create listener: %v", err) + ts.T().FailNow() + } + reHost, rePort, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + ts.T().FailNow() + } + go func() { + var opts []grpc.ServerOption + s := grpc.NewServer(opts...) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.privateKeyList}) + reflection.Register(s) + if err := s.Serve(lis); err != nil { + log.Errorf("failed to start GRPC server: %v", err) + ts.T().Fail() + } + }() + + rePortInt, err := strconv.Atoi(rePort) + if err != nil { + ts.T().FailNow() + } + + ts.app.Conf.RotateKey.Grpc = config.Grpc{ + Host: reHost, + Port: rePortInt, + Timeout: 30, + } + + ts.T().Log("suite setup completed") +} + +// ReencryptHeader serves a mock response since we don't need to test the actual reencryption +func (s *server) ReencryptHeader(ctx context.Context, req *re.ReencryptRequest) (*re.ReencryptResponse, error) { + // Mock response based on your needs + if req.Publickey == "phail" { + return &re.ReencryptResponse{}, errors.New("bad error") + } + + mockedResponse := &re.ReencryptResponse{ + Header: []byte("predefined header response"), + } + + return mockedResponse, nil +} + +func (ts *TestSuite) TestReEncryptHeader() { + fileID := ts.corrID + + for _, test := range []struct { + corrID string + expectedError error + expectedMgs string + expectedRes string + fileID string + testName string + }{ + { + testName: "ingested file", + expectedError: nil, + expectedMgs: "", + expectedRes: "ack", + corrID: ts.corrID, + fileID: ts.fileID, + }, + { + testName: "un-ingested file", + expectedError: errors.New("sql: no rows in result set"), + expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID), + expectedRes: "ackSendToError", + corrID: uuid.New().String(), + fileID: fileID, + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + res, msg, err := ts.app.reEncryptHeader(test.corrID, test.fileID) + assert.Equal(t, res, test.expectedRes) + assert.Equal(t, msg, test.expectedMgs) + assert.Equal(t, err, test.expectedError) + }) + } +} From 937ca6e4071b1dedf99050435a769038cb3bf0eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:02:29 +0000 Subject: [PATCH 123/184] Bump the all-modules group in /sda-sftp-inbox with 2 updates Bumps the all-modules group in /sda-sftp-inbox with 2 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [net.logstash.logback:logstash-logback-encoder](https://github.com/logfellow/logstash-logback-encoder). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.6 to 3.5.7 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.6...v3.5.7) Updates `net.logstash.logback:logstash-logback-encoder` from 8.1 to 9.0 - [Release notes](https://github.com/logfellow/logstash-logback-encoder/releases) - [Commits](https://github.com/logfellow/logstash-logback-encoder/compare/logstash-logback-encoder-8.1...logstash-logback-encoder-9.0) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: net.logstash.logback:logstash-logback-encoder dependency-version: '9.0' dependency-type: direct:production update-type: version-update:semver-major dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 105380057..de0756d39 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.6 + 3.5.7 @@ -118,7 +118,7 @@ net.logstash.logback logstash-logback-encoder - 8.1 + 9.0 org.bouncycastle From 796d9f3b4e3f0d6cc4ae5a4dae6a115b47be71fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 20:11:15 +0000 Subject: [PATCH 124/184] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [net.logstash.logback:logstash-logback-encoder](https://github.com/logfellow/logstash-logback-encoder). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.6 to 3.5.7 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.6...v3.5.7) Updates `net.logstash.logback:logstash-logback-encoder` from 8.1 to 9.0 - [Release notes](https://github.com/logfellow/logstash-logback-encoder/releases) - [Commits](https://github.com/logfellow/logstash-logback-encoder/compare/logstash-logback-encoder-8.1...logstash-logback-encoder-9.0) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: net.logstash.logback:logstash-logback-encoder dependency-version: '9.0' dependency-type: direct:production update-type: version-update:semver-major dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 969205b6a..187dc337d 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.6 + 3.5.7 no.uio.ifi @@ -89,7 +89,7 @@ net.logstash.logback logstash-logback-encoder - 8.1 + 9.0 org.springframework.boot From 86fcb4ae3ae2c01d1b507f1bf9cb482e11746a98 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Tue, 28 Oct 2025 16:06:41 +0100 Subject: [PATCH 125/184] Minor linter fix --- sda/internal/database/db_functions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index e131452b4..60da6516d 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1593,7 +1593,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "archived") assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileID2) - + db.Close() } From 1b53e8539c51b2b0dcfb8acd5917ec9979af11de Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 29 Oct 2025 09:06:09 +0000 Subject: [PATCH 126/184] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 537513d4f..2b3d50db3 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.20 -appVersion: v3.0.14 +version: 2.0.21 +appVersion: v3.0.26 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 70527b36e..a8abe33d8 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.20 -appVersion: v3.0.14 +version: 2.0.21 +appVersion: v3.0.26 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index e38f74dd1..6a1ec3ddb 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.13 -appVersion: v3.0.14 +version: 3.0.14 +appVersion: v3.0.26 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From e48edf793988b3be17326c34a795a690375f16a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 19:06:16 +0000 Subject: [PATCH 127/184] Bump the all-modules group in /sda-sftp-inbox with 2 updates Bumps the all-modules group in /sda-sftp-inbox with 2 updates: [org.junit:junit-bom](https://github.com/junit-team/junit-framework) and [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java). Updates `org.junit:junit-bom` from 6.0.0 to 6.0.1 - [Release notes](https://github.com/junit-team/junit-framework/releases) - [Commits](https://github.com/junit-team/junit-framework/compare/r6.0.0...r6.0.1) Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.792 to 1.12.793 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.792...1.12.793) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 6.0.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.793 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index de0756d39..423f5c3e0 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 6.0.0 + 6.0.1 pom import @@ -108,7 +108,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.792 + 1.12.793 com.google.guava From 079a1a4e2bcb24e10dfc9165bc384cd940541e10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Nov 2025 19:15:34 +0000 Subject: [PATCH 128/184] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp). Updates `no.elixir:crypt4gh` from 3.0.35 to 3.0.36 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.35...crypt4gh-3.0.36) Updates `com.squareup.okhttp3:okhttp-jvm` from 5.2.1 to 5.3.0 - [Changelog](https://github.com/square/okhttp/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okhttp/compare/parent-5.2.1...parent-5.3.0) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.36 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.squareup.okhttp3:okhttp-jvm dependency-version: 5.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 187dc337d..e5005426d 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.35 + 3.0.36 org.slf4j @@ -137,7 +137,7 @@ com.squareup.okhttp3 okhttp-jvm - 5.2.1 + 5.3.0 From 611684d107fd7debf29fad8356f49e9f1ed7492b Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Oct 2025 17:51:38 +0200 Subject: [PATCH 129/184] feat: add middleware for checking client version fix: c.Next() is run only at the last middleware call feat: use semver to check if minimum version req --- sda-download/api/middleware/middleware.go | 91 +++++++++++++++++++++++ sda-download/internal/config/config.go | 7 ++ 2 files changed, 98 insertions(+) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index de6a9846d..8e13c8f76 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -1,8 +1,11 @@ package middleware import ( + "fmt" "net/http" + "strings" + "github.com/Masterminds/semver/v3" "github.com/gin-gonic/gin" "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/session" @@ -85,6 +88,94 @@ func TokenMiddleware() gin.HandlerFunc { } } +// ClientVersionMiddleware checks for the required "sda-cli-version" header. +// It aborts the request with 412 (Precondition Failed) if the header is missing or +// if the version does not meet the minimum required version. +func ClientVersionMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + clientVersionStr := c.GetHeader("sda-cli-version") + expectedVersionStr := config.Config.App.ExpectedCliVersion + + // 1. Check if the header is present + if clientVersionStr == "" { + errorMessage := fmt.Sprintf( + "Error: Missing required header '%s'. Please ensure you are using the latest sda-cli client.", + "sda-cli-version", + ) + log.Warnf("request blocked (412): Missing required header '%s'", "sda-cli-version") + c.String(http.StatusPreconditionFailed, errorMessage) + c.AbortWithStatus(http.StatusPreconditionFailed) + + return + } + + // Safely strip the common 'v' prefix before SemVer parsing. + processedClientVersionStr := clientVersionStr + if after, ok :=strings.CutPrefix(processedClientVersionStr, "v"); ok { + processedClientVersionStr = after + } + + // Parse the expected minimum version from config + requiredVersion, err := semver.NewVersion(expectedVersionStr) + if err != nil { + log.Errorf("configuration error: cannot parse expected minimum version '%s' as SemVer: %v. Blocking request.", expectedVersionStr, err) + c.String(http.StatusInternalServerError, "Internal Server Error: Invalid minimum client version configured.") + c.AbortWithStatus(http.StatusInternalServerError) + + return + } + + // Parse the client's provided version (using the processed string) + clientVersion, err := semver.NewVersion(processedClientVersionStr) + if err != nil { + log.Warnf("request blocked (412): processed client version header '%s' is not a valid semantic version: %v", processedClientVersionStr, err) + errorMessage := fmt.Sprintf( + "Error: Your sda-cli client version ('%s') is invalid. Required minimum version is '%s'.", + clientVersionStr, + expectedVersionStr, + ) + c.String(http.StatusPreconditionFailed, errorMessage) + c.AbortWithStatus(http.StatusPreconditionFailed) + + return + } + + // 2. Check if the client version is sufficient (clientVersion >= requiredVersion) + if clientVersion.LessThan(requiredVersion) { + errorMessage := fmt.Sprintf( + "Error: Your sda-cli client version ('%s') is insufficient. Please update to at least version '%s' to proceed.", + clientVersionStr, + expectedVersionStr, + ) + log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, expectedVersionStr) + c.String(http.StatusPreconditionFailed, errorMessage) + c.AbortWithStatus(http.StatusPreconditionFailed) + + return + } + + // Version is correct, proceed to the next handler/middleware + log.Debugf("client version check passed: %s", clientVersionStr) + } +} + +// ChainDefaultMiddleware chains the ClientVersionMiddleware and TokenMiddleware. +// It is intended to be the default composite middleware set for the application. +func ChainDefaultMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + // 1. Run the Client Version Check. This will abort if version is invalid/missing (HTTP 412) + ClientVersionMiddleware()(c) + + // Check if the request was aborted by the version middleware + if c.IsAborted() { + return + } + + // 2. Run the Token Middleware (only if version check passed) + TokenMiddleware()(c) + } +} + // GetCacheFromContext is a helper function that endpoints can use to get data // stored to the *current* request context (not the session storage). // The request context was populated by the middleware, which in turn uses the session storage. diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 2e9a7129c..30265b9b8 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -60,6 +60,11 @@ type AppConfig struct { // Selected middleware for authentication and authorizaton // Optional. Default value is "default" for TokenMiddleware Middleware string + + // Expected version string for the sda-cli client (e.g., "v1.2.3") + // If the client version header does not match this, the request is blocked. + // Optional. Default value is "v0.0.0" + ExpectedCliVersion string } // Stores the Crypt4GH private key used internally @@ -242,6 +247,7 @@ func (c *Map) applyDefaults() { viper.SetDefault("app.host", "0.0.0.0") viper.SetDefault("app.port", 8080) viper.SetDefault("app.middleware", "default") + viper.SetDefault("app.expectedcliversion", "v0.0.0") viper.SetDefault("session.expiration", -1) viper.SetDefault("session.secure", true) viper.SetDefault("session.httponly", true) @@ -370,6 +376,7 @@ func (c *Map) appConfig() error { c.App.ServerCert = viper.GetString("app.servercert") c.App.ServerKey = viper.GetString("app.serverkey") c.App.Middleware = viper.GetString("app.middleware") + c.App.ExpectedCliVersion = viper.GetString("app.expectedcliversion") if c.App.Port != 443 && c.App.Port != 8080 { c.App.Port = viper.GetInt("app.port") From bf7c84d55b00535dead423b90fdac192f3a22a31 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Oct 2025 17:52:24 +0200 Subject: [PATCH 130/184] feat: update default SelectedMiddleware --- sda-download/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 11734022d..d9113c541 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -28,7 +28,7 @@ func init() { // nolint:gocritic // this nolint can be removed, if you have more than one middlewares available switch conf.App.Middleware { //nolint:revive default: - api.SelectedMiddleware = middleware.TokenMiddleware + api.SelectedMiddleware = middleware.ChainDefaultMiddleware } log.Infof("%s middleware selected", conf.App.Middleware) From b6237e381ab3e588e75e87954c0bf93fd77cce97 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Oct 2025 23:07:59 +0200 Subject: [PATCH 131/184] feat: update go mod --- sda-download/go.mod | 1 + sda-download/go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/sda-download/go.mod b/sda-download/go.mod index b5e36f4b9..c94c74b6c 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -4,6 +4,7 @@ go 1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 + github.com/Masterminds/semver/v3 v3.4.0 github.com/aws/aws-sdk-go v1.55.8 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.11.0 diff --git a/sda-download/go.sum b/sda-download/go.sum index 111583084..8ad3a676e 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -2,6 +2,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= From f75688da56d944fa2af23340c926990fbed80f73 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Oct 2025 23:37:16 +0200 Subject: [PATCH 132/184] feat: add unit tests for ClientVersionMiddleware --- .../api/middleware/middleware_test.go | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index ed680004d..e7731e554 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" "github.com/gin-gonic/gin" @@ -303,3 +304,109 @@ func TestGetDatasets(t *testing.T) { t.Errorf("TestStoreDatasets failed, got %s, expected %s", storedDatasets, datasets) } } + +func TestClientVersionMiddleware(t *testing.T) { + originalExpectedCliVersion := config.Config.App.ExpectedCliVersion + defer func() { + config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + }() + + const headerName = "sda-cli-version" + + tests := []struct { + name string + clientVersionHeader string + configExpectedVersion string + expectedStatus int + expectedBodyContains string + }{ + { + name: "Fail_MissingHeader", + clientVersionHeader: "", + configExpectedVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "Missing required header", + }, + { + name: "Fail_InvalidClientSemVer", + clientVersionHeader: "v-invalid-1", + configExpectedVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is invalid", + }, + { + name: "Fail_InsufficientVersion", + clientVersionHeader: "v0.1.9", + configExpectedVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", + }, + { + // This tests the logic for handling a bad configuration value + name: "Fail_InvalidConfigVersion", + clientVersionHeader: "v0.2.0", + configExpectedVersion: "not-semver", + expectedStatus: http.StatusInternalServerError, // 500 + expectedBodyContains: "Internal Server Error: Invalid minimum client version configured.", + }, + { + name: "Success_EqualVersion", + clientVersionHeader: "v0.2.0", + configExpectedVersion: "v0.2.0", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", + }, + { + name: "Success_NewerVersion", + clientVersionHeader: "v0.3.0", + configExpectedVersion: "v0.2.0", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Setup + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + _, router := gin.CreateTestContext(w) + + // Set the configuration mock and the request header + config.Config.App.ExpectedCliVersion = tt.configExpectedVersion + if tt.clientVersionHeader != "" { + r.Header.Set(headerName, tt.clientVersionHeader) + } + + // Define a dummy handler to check if the middleware allowed passage + var passed bool + dummyHandler := func(c *gin.Context) { + passed = true + c.Status(http.StatusOK) // Explicitly set OK status if allowed to pass + } + + // Send request through the middleware + router.GET("/", ClientVersionMiddleware(), dummyHandler) + router.ServeHTTP(w, r) + + // Assertion 1: Check Status Code + if w.Code != tt.expectedStatus { + t.Errorf("status code mismatch.\nGot: %d\nWant: %d", w.Code, tt.expectedStatus) + } + + // Assertion 2: Check Body Content for Failures + body := w.Body.String() + if tt.expectedStatus != http.StatusOK && !strings.Contains(body, tt.expectedBodyContains) { + t.Errorf("response body mismatch.\nGot Body: %s\nWant Body to contain: %s", body, tt.expectedBodyContains) + } + + // Assertion 3: Check if the request was allowed to pass (only for success cases) + if tt.expectedStatus == http.StatusOK && !passed { + t.Error("success case failed: Middleware unexpectedly blocked the request.") + } + if tt.expectedStatus != http.StatusOK && passed { + t.Error("failure case failed: Middleware unexpectedly allowed the request to pass.") + } + }) + } +} From de1496e6aea0d563e942e8033042f4fde8d9b386 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Oct 2025 23:51:39 +0200 Subject: [PATCH 133/184] feat: add unit tests for ChainDefaultMiddleware --- .../api/middleware/middleware_test.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index e7731e554..cfa4c2584 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -410,3 +410,103 @@ func TestClientVersionMiddleware(t *testing.T) { }) } } + +func TestChainDefaultMiddleware_Success(t *testing.T) { + // Setup global mocks required for TokenMiddleware Success (No Cache) + originalGetToken := auth.GetToken + originalGetVisas := auth.GetVisas + originalGetPermissions := auth.GetPermissions + originalNewSessionKey := session.NewSessionKey + originalSessionName := config.Config.Session.Name + + auth.GetToken = func(_ http.Header) (string, int, error) { return token, 200, nil } + auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { return &auth.Visas{}, nil } + auth.GetPermissions = func(_ auth.Visas) []string { return []string{"dataset1"} } + session.NewSessionKey = func() string { return "key" } + config.Config.Session.Name = "sda_session_key" // Set session name for cookie assertion + + defer func() { + auth.GetToken = originalGetToken + auth.GetVisas = originalGetVisas + auth.GetPermissions = originalGetPermissions + session.NewSessionKey = originalNewSessionKey + config.Config.Session.Name = originalSessionName + }() + + // Setup config for ClientVersionMiddleware Success + originalExpectedCliVersion := config.Config.App.ExpectedCliVersion + config.Config.App.ExpectedCliVersion = "v0.2.0" + defer func() { + config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + }() + + // Setup Request/Response + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("sda-cli-version", "v0.3.0") // Newer version, should pass + _, router := gin.CreateTestContext(w) + + // Send request through the chain middleware + router.GET("/", ChainDefaultMiddleware(), testEndpoint) + router.ServeHTTP(w, r) + + // Assertions + expectedStatusCode := http.StatusOK // Both middlewares passed + if w.Code != expectedStatusCode { + t.Errorf("TestChainDefaultMiddleware_Success failed, got status %d expected %d", w.Code, expectedStatusCode) + } + // Check that a session cookie was set by TokenMiddleware (confirming it ran) + cookies := w.Result().Cookies() + cookieFound := false + for _, c := range cookies { + if c.Name == "sda_session_key" { + cookieFound = true + break + } + } + + if !cookieFound { + t.Error("TestChainDefaultMiddleware_Success failed, expected a session cookie, but none was found.") + } +} + +func TestChainDefaultMiddleware_Fail_VersionAbortsChain(t *testing.T) { + // We use a flag to assert that TokenMiddleware was NOT executed. + originalGetToken := auth.GetToken + wasTokenCalled := false + auth.GetToken = func(_ http.Header) (string, int, error) { + wasTokenCalled = true + + return token, 200, nil + } + defer func() { + auth.GetToken = originalGetToken + }() + + // Setup config for ClientVersionMiddleware Failure (Insufficient version) + originalExpectedCliVersion := config.Config.App.ExpectedCliVersion + config.Config.App.ExpectedCliVersion = "v0.2.0" + defer func() { + config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + }() + + // Setup Request/Response + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + r.Header.Set("sda-cli-version", "0.1.0") // Insufficient version, should abort (412) + _, router := gin.CreateTestContext(w) + + // Send request through the chain middleware + router.GET("/", ChainDefaultMiddleware(), testEndpoint) + router.ServeHTTP(w, r) + + // Assertions + expectedStatusCode := http.StatusPreconditionFailed // 412 + if w.Code != expectedStatusCode { + t.Errorf("TestChainDefaultMiddleware_Fail_VersionAbortsChain failed, got status %d expected %d", w.Code, expectedStatusCode) + } + + if wasTokenCalled { + t.Error("TestChainDefaultMiddleware_Fail_VersionAbortsChain failed, TokenMiddleware was executed when it should have been aborted.") + } +} From 3a5bc7e499ed3f5984a78d96791bcfa3b15b28e6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 12:04:35 +0200 Subject: [PATCH 134/184] refactor: parse expectedClientVersion in the config --- sda-download/api/middleware/middleware.go | 43 +++++++---------------- sda-download/internal/config/config.go | 14 ++++++-- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 8e13c8f76..7040d32d3 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -3,7 +3,6 @@ package middleware import ( "fmt" "net/http" - "strings" "github.com/Masterminds/semver/v3" "github.com/gin-gonic/gin" @@ -88,51 +87,35 @@ func TokenMiddleware() gin.HandlerFunc { } } -// ClientVersionMiddleware checks for the required "sda-cli-version" header. +// ClientVersionMiddleware checks for the required "SDA-Client-Version" header. // It aborts the request with 412 (Precondition Failed) if the header is missing or // if the version does not meet the minimum required version. func ClientVersionMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - clientVersionStr := c.GetHeader("sda-cli-version") - expectedVersionStr := config.Config.App.ExpectedCliVersion + const headerName = "SDA-Client-Version" + clientVersionStr := c.GetHeader(headerName) // 1. Check if the header is present if clientVersionStr == "" { errorMessage := fmt.Sprintf( "Error: Missing required header '%s'. Please ensure you are using the latest sda-cli client.", - "sda-cli-version", + headerName, ) - log.Warnf("request blocked (412): Missing required header '%s'", "sda-cli-version") + log.Warnf("request blocked (412): Missing required header '%s'", headerName) c.String(http.StatusPreconditionFailed, errorMessage) c.AbortWithStatus(http.StatusPreconditionFailed) return } - // Safely strip the common 'v' prefix before SemVer parsing. - processedClientVersionStr := clientVersionStr - if after, ok :=strings.CutPrefix(processedClientVersionStr, "v"); ok { - processedClientVersionStr = after - } - - // Parse the expected minimum version from config - requiredVersion, err := semver.NewVersion(expectedVersionStr) - if err != nil { - log.Errorf("configuration error: cannot parse expected minimum version '%s' as SemVer: %v. Blocking request.", expectedVersionStr, err) - c.String(http.StatusInternalServerError, "Internal Server Error: Invalid minimum client version configured.") - c.AbortWithStatus(http.StatusInternalServerError) - - return - } - // Parse the client's provided version (using the processed string) - clientVersion, err := semver.NewVersion(processedClientVersionStr) + clientVersion, err := semver.NewVersion(clientVersionStr) if err != nil { - log.Warnf("request blocked (412): processed client version header '%s' is not a valid semantic version: %v", processedClientVersionStr, err) + log.Warnf("request blocked (412): processed client version header '%s' is not a valid semantic version: %v", clientVersionStr, err) errorMessage := fmt.Sprintf( - "Error: Your sda-cli client version ('%s') is invalid. Required minimum version is '%s'.", + "Error: Your sda-cli client version '%s' is invalid. Required minimum version is '%s'.", clientVersionStr, - expectedVersionStr, + config.Config.App.ExpectedCliVersionStr, ) c.String(http.StatusPreconditionFailed, errorMessage) c.AbortWithStatus(http.StatusPreconditionFailed) @@ -141,13 +124,13 @@ func ClientVersionMiddleware() gin.HandlerFunc { } // 2. Check if the client version is sufficient (clientVersion >= requiredVersion) - if clientVersion.LessThan(requiredVersion) { + if clientVersion.LessThan(config.Config.App.ExpectedCliVersion) { errorMessage := fmt.Sprintf( - "Error: Your sda-cli client version ('%s') is insufficient. Please update to at least version '%s' to proceed.", + "Error: Your sda-cli client version '%s' is insufficient. Please update to at least version '%s' to proceed.", clientVersionStr, - expectedVersionStr, + config.Config.App.ExpectedCliVersionStr, ) - log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, expectedVersionStr) + log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.ExpectedCliVersionStr) c.String(http.StatusPreconditionFailed, errorMessage) c.AbortWithStatus(http.StatusPreconditionFailed) diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 30265b9b8..7a025f531 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sda-download/internal/storage" @@ -64,7 +65,8 @@ type AppConfig struct { // Expected version string for the sda-cli client (e.g., "v1.2.3") // If the client version header does not match this, the request is blocked. // Optional. Default value is "v0.0.0" - ExpectedCliVersion string + ExpectedCliVersion *semver.Version + ExpectedCliVersionStr string // This is the original string from the config file } // Stores the Crypt4GH private key used internally @@ -376,7 +378,14 @@ func (c *Map) appConfig() error { c.App.ServerCert = viper.GetString("app.servercert") c.App.ServerKey = viper.GetString("app.serverkey") c.App.Middleware = viper.GetString("app.middleware") - c.App.ExpectedCliVersion = viper.GetString("app.expectedcliversion") + + // Validate and parse the configured minimum client version into a SemVer object + c.App.ExpectedCliVersionStr = viper.GetString("app.expectedcliversion") + parsedVersion, err := semver.NewVersion(c.App.ExpectedCliVersionStr) + if err != nil { + return fmt.Errorf("app.expectedcliversion value='%s' is not a valid semantic version: %v", c.App.ExpectedCliVersionStr, err) + } + c.App.ExpectedCliVersion = parsedVersion if c.App.Port != 443 && c.App.Port != 8080 { c.App.Port = viper.GetInt("app.port") @@ -384,7 +393,6 @@ func (c *Map) appConfig() error { c.App.Port = 443 } - var err error if viper.GetString("c4gh.transientKeyPath") != "" { if !viper.IsSet("c4gh.transientPassphrase") { return errors.New("c4gh.transientPassphrase is not set") From f50f88758c7b3f56a7066cfdbc82eb28dc2abf69 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 12:05:01 +0200 Subject: [PATCH 135/184] feat: update unit tests --- .../api/middleware/middleware_test.go | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index cfa4c2584..bc60758d5 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/Masterminds/semver/v3" "github.com/gin-gonic/gin" "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/session" @@ -311,8 +312,6 @@ func TestClientVersionMiddleware(t *testing.T) { config.Config.App.ExpectedCliVersion = originalExpectedCliVersion }() - const headerName = "sda-cli-version" - tests := []struct { name string clientVersionHeader string @@ -341,14 +340,6 @@ func TestClientVersionMiddleware(t *testing.T) { expectedStatus: http.StatusPreconditionFailed, // 412 expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", }, - { - // This tests the logic for handling a bad configuration value - name: "Fail_InvalidConfigVersion", - clientVersionHeader: "v0.2.0", - configExpectedVersion: "not-semver", - expectedStatus: http.StatusInternalServerError, // 500 - expectedBodyContains: "Internal Server Error: Invalid minimum client version configured.", - }, { name: "Success_EqualVersion", clientVersionHeader: "v0.2.0", @@ -372,10 +363,16 @@ func TestClientVersionMiddleware(t *testing.T) { r := httptest.NewRequest("GET", "/", nil) _, router := gin.CreateTestContext(w) - // Set the configuration mock and the request header - config.Config.App.ExpectedCliVersion = tt.configExpectedVersion + config.Config.App.ExpectedCliVersionStr = tt.configExpectedVersion + // Set the configuration mock by parsing the string into the required SemVer object + parsedVersion, err := semver.NewVersion(tt.configExpectedVersion) + if err != nil { + t.Fatalf("Test setup error: Failed to parse expected version '%s': %v", tt.configExpectedVersion, err) + } + config.Config.App.ExpectedCliVersion = parsedVersion + if tt.clientVersionHeader != "" { - r.Header.Set(headerName, tt.clientVersionHeader) + r.Header.Set("SDA-Client-Version", tt.clientVersionHeader) } // Define a dummy handler to check if the middleware allowed passage @@ -435,7 +432,11 @@ func TestChainDefaultMiddleware_Success(t *testing.T) { // Setup config for ClientVersionMiddleware Success originalExpectedCliVersion := config.Config.App.ExpectedCliVersion - config.Config.App.ExpectedCliVersion = "v0.2.0" + expectedCliVersion, err := semver.NewVersion("0.2.0") + if err != nil { + t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) + } + config.Config.App.ExpectedCliVersion = expectedCliVersion defer func() { config.Config.App.ExpectedCliVersion = originalExpectedCliVersion }() @@ -443,7 +444,7 @@ func TestChainDefaultMiddleware_Success(t *testing.T) { // Setup Request/Response w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/", nil) - r.Header.Set("sda-cli-version", "v0.3.0") // Newer version, should pass + r.Header.Set("SDA-Client-Version", "v0.3.0") // Newer version, should pass _, router := gin.CreateTestContext(w) // Send request through the chain middleware @@ -485,7 +486,11 @@ func TestChainDefaultMiddleware_Fail_VersionAbortsChain(t *testing.T) { // Setup config for ClientVersionMiddleware Failure (Insufficient version) originalExpectedCliVersion := config.Config.App.ExpectedCliVersion - config.Config.App.ExpectedCliVersion = "v0.2.0" + expectedCliVersion, err := semver.NewVersion("0.2.0") + if err != nil { + t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) + } + config.Config.App.ExpectedCliVersion = expectedCliVersion defer func() { config.Config.App.ExpectedCliVersion = originalExpectedCliVersion }() @@ -493,7 +498,7 @@ func TestChainDefaultMiddleware_Fail_VersionAbortsChain(t *testing.T) { // Setup Request/Response w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/", nil) - r.Header.Set("sda-cli-version", "0.1.0") // Insufficient version, should abort (412) + r.Header.Set("SDA-Client-Version", "0.1.0") // Insufficient version, should abort (412) _, router := gin.CreateTestContext(w) // Send request through the chain middleware From 4a9845f27874f09a0d8032b4a4da0d5237d9c71d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 15:06:52 +0200 Subject: [PATCH 136/184] feat: fix integration test for sda-download --- .../tests/common/50_check_endpoint.sh | 22 +++++++++---------- .../tests/common/70_check_download.sh | 4 ++-- .../tests/common/80_check_reencrypt.sh | 6 ++--- .../tests/common/90_check_s3_errors.sh | 20 ++++++++--------- .../tests/s3notls/52_check_endpoint.sh | 20 ++++++++--------- sda-download/dev_utils/config-notls.yaml | 3 +++ sda-download/dev_utils/config.yaml | 1 + 7 files changed, 40 insertions(+), 36 deletions(-) diff --git a/sda-download/.github/integration/tests/common/50_check_endpoint.sh b/sda-download/.github/integration/tests/common/50_check_endpoint.sh index 5a985fc8d..48d62d044 100755 --- a/sda-download/.github/integration/tests/common/50_check_endpoint.sh +++ b/sda-download/.github/integration/tests/common/50_check_endpoint.sh @@ -33,7 +33,7 @@ echo "Head method health endpoint is ok" # ------------------ # Test empty token -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem https://localhost:8443/metadata/datasets) +check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) if [ "$check_401" != "401" ]; then echo "no token provided should give 401" @@ -43,7 +43,7 @@ fi echo "got correct response when no token provided" -check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem https://localhost:8443/metadata/datasets) +check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) if [ "$check_405" != "405" ]; then echo "POST should not be allowed" @@ -60,7 +60,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' ## Test datasets endpoint -check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]') +check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets | jq -r '.[0]') if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" @@ -72,7 +72,7 @@ echo "expected dataset found" ## Test datasets/files endpoint -check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') +check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') if [ "$check_files" != "urn:neic:001-002" ]; then echo "file with id urn:neic:001-002 not found" @@ -90,7 +90,7 @@ export C4GH_PASSPHRASE crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002" --output test-download.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002" --output test-download.txt cmp --silent old-file.txt test-download.txt status=$? @@ -101,7 +101,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt @@ -114,7 +114,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt @@ -127,7 +127,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt @@ -142,7 +142,7 @@ fi # test that downloads of decrypted files from a download instance that # serves only encrypted files (here running at port 8443) should fail -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download-fail.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/files/urn:neic:001-002" --output test-download-fail.txt if ! grep -q "downloading unencrypted data is not supported" test-download-fail.txt; then echo "got unexpected response when trying to download unencrypted data from encrypted endpoint" @@ -156,7 +156,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' ## Test datasets endpoint -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets) +check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) if [ "$check_empty_token" != "200" ]; then echo "response for empty token is not 200" @@ -174,7 +174,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' ## Test datasets endpoint -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets) +check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) if [ "$check_empty_token" != "200" ]; then echo "response for token with untrusted sources is not 200" diff --git a/sda-download/.github/integration/tests/common/70_check_download.sh b/sda-download/.github/integration/tests/common/70_check_download.sh index 2e5c320c8..f510065c4 100644 --- a/sda-download/.github/integration/tests/common/70_check_download.sh +++ b/sda-download/.github/integration/tests/common/70_check_download.sh @@ -15,7 +15,7 @@ C4GH_PASSPHRASE=$(yq .c4gh.passphrase config.yaml) export C4GH_PASSPHRASE # download decrypted full file, check file size -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/s3/$dataset/$file" --output full1.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" --output full1.bam file_size=$(stat -c %s full1.bam) # Get the size of the file if [ "$file_size" -ne "$expected_size" ]; then @@ -24,7 +24,7 @@ if [ "$file_size" -ne "$expected_size" ]; then fi # test that start, end=0 returns the whole file -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam if ! cmp --silent full1.bam full2.bam; then echo "Full decrypted files, with and without coordinates, are different" diff --git a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh index 8354a8e8f..2f90949e3 100644 --- a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh +++ b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh @@ -19,7 +19,7 @@ file="dummy_data" expected_size=1048605 # Download unencrypted full file (from download service at port 9443), check file size -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/s3/$dataset/$file" --output full1.bam +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" --output full1.bam file_size=$(stat -c %s full1.bam) # Get the size of the file if [ "$file_size" -ne "$expected_size" ]; then @@ -30,7 +30,7 @@ fi # Test reencrypt the file header with the client public key clientkey=$(base64 -w0 client.pub.pem) reencryptedFile=reencrypted.bam.c4gh -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3/$dataset/$file" --output $reencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" --output $reencryptedFile expected_encrypted_size=1049205 file_size=$(stat -c %s $reencryptedFile) @@ -55,7 +55,7 @@ fi # download reencrypted partial file, check file size partReencryptedFile=part1.bam.c4gh -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile file_size=$(stat -c %s $partReencryptedFile) # Get the size of the file part_expected_size=65688 diff --git a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh index a0413a246..0c5cefe13 100644 --- a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh +++ b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh @@ -22,39 +22,39 @@ bad_token=BADeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJyZXF1ZXN0ZXJAZGVtby # Test error codes and error messages returned to the user # try to download encrypted file without sending a public key -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") if ! echo "$resp" | grep -q "c4gh public key is missing from the header"; then echo "Incorrect response, expected 'c4gh public key is missing from the header' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 400 ]; then echo "Incorrect response with missing public key, expected 400 got $resp" exit 1 fi # try to download encrypted file with a bad public key -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") if ! echo "$resp" | grep -q "file re-encryption error"; then echo "Incorrect response, expected 'file re-encryption error' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 500 ]; then echo "Incorrect response with missing public key, expected 500 got $resp" fi # try to download encrypted file from instance that serves unencrypted files -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:9443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file") if ! echo "$resp" | grep -q "downloading encrypted data is not supported"; then echo "Incorrect response, expected 'downloading encrypted data is not supported' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:9443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 400 ]; then echo "Incorrect response, expected 400 got $resp" exit 1 @@ -63,13 +63,13 @@ fi # try to download a file the user doesn't have access to -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") if ! echo "$resp" | grep -q "get visas failed"; then echo "Incorrect response, expected 'get visas failed' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 401 ]; then echo "Incorrect response, expected 401 got $resp" exit 1 @@ -77,13 +77,13 @@ fi # try to download a file that does not exist -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:9443/s3/$dataset/nonexistentfile") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/nonexistentfile") if [ -n "$resp" ]; then echo "Incorrect response, expected no error message, got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" "https://localhost:9443/s3/$dataset/nonexistentfile" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/nonexistentfile" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 404 ]; then echo "Incorrect response, expected 404 got $resp" exit 1 diff --git a/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh b/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh index a065a2ec3..e70c08f63 100644 --- a/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh +++ b/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh @@ -18,7 +18,7 @@ echo "Health endpoint is ok" # ------------------ # Test empty token -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" http://localhost:8080/metadata/datasets) +check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -H "SDA-Client-Version: v0.3.0" http://localhost:8080/metadata/datasets) if [ "$check_401" != "401" ]; then echo "no token provided should give 401" @@ -28,7 +28,7 @@ fi echo "got correct response when no token provided" -check_405=$(curl -X POST -o /dev/null -s -w "%{http_code}\n" http://localhost:8080/metadata/datasets ) +check_405=$(curl -X POST -o /dev/null -s -w "%{http_code}\n" -H "SDA-Client-Version: v0.3.0" http://localhost:8080/metadata/datasets) if [ "$check_405" != "405" ]; then echo "POST should not be allowed" @@ -45,7 +45,7 @@ token=$(curl -s "http://localhost:8000/tokens" | jq -r '.[0]') ## Test datasets endpoint -check_dataset=$(curl -s -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]') +check_dataset=$(curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" http://localhost:8080/metadata/datasets | jq -r '.[0]') if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" @@ -57,7 +57,7 @@ echo "expected dataset found" ## Test datasets/files endpoint -check_files=$(curl -s -H "Authorization: Bearer $token" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') +check_files=$(curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') if [ "$check_files" != "urn:neic:001-002" ]; then echo "file with id urn:neic:001-002 not found" @@ -76,7 +76,7 @@ export C4GH_PASSPHRASE crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt # first try downloading from download instance serving encrypted data, should fail -curl -s -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt if ! grep -q "downloading unencrypted data is not supported" "test-download.txt"; then echo "wrong response when trying to download unencrypted data from encrypted endpoint" @@ -84,7 +84,7 @@ if ! grep -q "downloading unencrypted data is not supported" "test-download.txt" fi # now try downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" "http://localhost:9080/files/urn:neic:001-002" --output test-download.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002" --output test-download.txt cmp --silent old-file.txt test-download.txt @@ -96,7 +96,7 @@ else fi # downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt @@ -110,7 +110,7 @@ else fi # downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt @@ -130,7 +130,7 @@ token=$(curl -s "http://localhost:8000/tokens" | jq -r '.[1]') ## Test datasets endpoint -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets) +check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" http://localhost:8080/metadata/datasets) if [ "$check_empty_token" != "200" ]; then echo "response for empty token is not 200" @@ -148,7 +148,7 @@ token=$(curl -s "http://localhost:8000/tokens" | jq -r '.[2]') ## Test datasets endpoint -check_dataset=$(curl -s -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]') +check_dataset=$(curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" http://localhost:8080/metadata/datasets | jq -r '.[0]') if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index f67d8d7f5..95e5117d8 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -1,3 +1,6 @@ +app: + expectedcliversion: "v0.2.0" + log: level: "debug" format: "json" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index a75f9bfec..f581487e8 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -4,6 +4,7 @@ app: serverkey: "./dev_utils/certs/download-key.pem" port: "8443" middleware: "default" + expectedcliversion: "v0.2.0" log: level: "debug" From 46847f239765ac040d4b7a1494ce097aa75a2866 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 15:25:02 +0200 Subject: [PATCH 137/184] fix: unit test for config --- sda-download/internal/config/config_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index e656301de..306999419 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/Masterminds/semver/v3" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" "github.com/spf13/viper" @@ -83,16 +84,27 @@ func (ts *TestSuite) TestAppConfig() { viper.Set("db.sslmode", "disable") viper.Set("app.middleware", "noexist") + viper.Set("app.expectedcliversion", "v0.2.0") + c := &Map{} err = c.appConfig() assert.Error(ts.T(), err, "Error expected") viper.Reset() + // Test fail on invalid expected client version + viper.Set("app.expectedcliversion", "not-a-semver") + c = &Map{} + err = c.appConfig() + assert.Error(ts.T(), err, "Error expected for invalid semver string") + assert.Contains(ts.T(), err.Error(), "'not-a-semver' is not a valid semantic version") + viper.Reset() + viper.Set("app.host", "test") viper.Set("app.port", 1234) viper.Set("app.servercert", "test") viper.Set("app.serverkey", "test") + viper.Set("app.expectedcliversion", "v0.2.0") viper.Set("log.logLevel", "debug") viper.Set("db.sslmode", "disable") viper.Set("c4gh.transientKeyPath", privateKeyFile.Name()) @@ -108,6 +120,11 @@ func (ts *TestSuite) TestAppConfig() { assert.NotEmpty(ts.T(), c.C4GH.PrivateKey) assert.NotEmpty(ts.T(), c.C4GH.PublicKeyB64) + // Assert ExpectedCliVersion + expectedVersion, _ := semver.NewVersion("v0.2.0") + assert.NotNil(ts.T(), c.App.ExpectedCliVersion, "ExpectedCliVersion should be parsed and not nil") + assert.True(ts.T(), expectedVersion.Equal(c.App.ExpectedCliVersion), "Parsed version does not match expected version") + // Check the private key that was loaded by checking the derived public key publicKey, err := base64.StdEncoding.DecodeString(c.C4GH.PublicKeyB64) assert.Nilf(ts.T(), err, "Incorrect public c4gh key generated (error in base64 encoding)") From 679ca1e33ffd84302d7725651c291ca75025d510 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 15:38:06 +0200 Subject: [PATCH 138/184] feat: add integration test for client version header --- .../tests/common/50_check_endpoint.sh | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/sda-download/.github/integration/tests/common/50_check_endpoint.sh b/sda-download/.github/integration/tests/common/50_check_endpoint.sh index 48d62d044..91b9fb07a 100755 --- a/sda-download/.github/integration/tests/common/50_check_endpoint.sh +++ b/sda-download/.github/integration/tests/common/50_check_endpoint.sh @@ -29,6 +29,51 @@ fi echo "Head method health endpoint is ok" +# ------------------ +# Test Client Version Header +# These tests verify that the version middleware runs before authentication. +# We assume the app is configured to require a minimum version (v0.2.0). + +# Fail - missing header (expected 412 Precondition Failed) +check_missing_header=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem "https://localhost:8443/metadata/datasets") + +if [ "$check_missing_header" != "412" ]; then + echo "Client Version Test FAIL: missing header should return 412" + echo "got: ${check_missing_header}" + exit 1 +fi +echo "Client Version Test OK: Missing header correctly returns 412" + +# Fail - insufficient version (e.g., v0.1.0, Expected 412) +check_insufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.1.0" "https://localhost:8443/metadata/datasets") + +if [ "$check_insufficient_version" != "412" ]; then + echo "Client Version Test FAIL: insufficient version (v0.1.0) should return 412" + echo "got: ${check_insufficient_version}" + exit 1 +fi +echo "Client Version Test OK: Insufficient version (v0.1.0) correctly returns 412" + + +# Success - sufficient version (e.g., v0.2.0, Expected 401 from token middleware) +check_sufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.2.0" "https://localhost:8443/metadata/datasets") + +if [ "$check_sufficient_version" != "401" ]; then + echo "Client Version Test FAIL: sufficient version (v0.2.0) passed version check but failed token check (Expected 401)" + echo "got: ${check_sufficient_version}" + exit 1 +fi +echo "Client Version Test OK: sufficient version (v0.2.0) correctly proceeds to token check (returns 401)" + +# Test 4: Success - newer version (e.g., v1.0.0, Expected 401 from token middleware) +check_newer_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v1.0.0" "https://localhost:8443/metadata/datasets") + +if [ "$check_newer_version" != "401" ]; then + echo "Client Version Test FAIL: Newer version (v1.0.0) passed version check but failed token check (Expected 401)" + echo "got: ${check_newer_version}" + exit 1 +fi +echo "Client Version Test OK: Newer version (v1.0.0) correctly proceeds to token check (returns 401)" # ------------------ # Test empty token From 762e5d746fc3b75a03e0e857278e40f068a2df23 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 15:47:27 +0200 Subject: [PATCH 139/184] fix: lint, gofmt for sda-download --- .../api/middleware/middleware_test.go | 45 ++++++++++--------- sda-download/internal/config/config.go | 2 +- sda-download/internal/config/config_test.go | 1 - 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index bc60758d5..7c79b4f19 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -320,39 +320,39 @@ func TestClientVersionMiddleware(t *testing.T) { expectedBodyContains string }{ { - name: "Fail_MissingHeader", - clientVersionHeader: "", + name: "Fail_MissingHeader", + clientVersionHeader: "", configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "Missing required header", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "Missing required header", }, { - name: "Fail_InvalidClientSemVer", - clientVersionHeader: "v-invalid-1", + name: "Fail_InvalidClientSemVer", + clientVersionHeader: "v-invalid-1", configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is invalid", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is invalid", }, { - name: "Fail_InsufficientVersion", - clientVersionHeader: "v0.1.9", + name: "Fail_InsufficientVersion", + clientVersionHeader: "v0.1.9", configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", }, { - name: "Success_EqualVersion", - clientVersionHeader: "v0.2.0", + name: "Success_EqualVersion", + clientVersionHeader: "v0.2.0", configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusOK, // 200 - expectedBodyContains: "", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", }, { - name: "Success_NewerVersion", - clientVersionHeader: "v0.3.0", + name: "Success_NewerVersion", + clientVersionHeader: "v0.3.0", configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusOK, // 200 - expectedBodyContains: "", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", }, } @@ -432,11 +432,11 @@ func TestChainDefaultMiddleware_Success(t *testing.T) { // Setup config for ClientVersionMiddleware Success originalExpectedCliVersion := config.Config.App.ExpectedCliVersion - expectedCliVersion, err := semver.NewVersion("0.2.0") + expectedCliVersion, err := semver.NewVersion("0.2.0") if err != nil { t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) } - config.Config.App.ExpectedCliVersion = expectedCliVersion + config.Config.App.ExpectedCliVersion = expectedCliVersion defer func() { config.Config.App.ExpectedCliVersion = originalExpectedCliVersion }() @@ -462,6 +462,7 @@ func TestChainDefaultMiddleware_Success(t *testing.T) { for _, c := range cookies { if c.Name == "sda_session_key" { cookieFound = true + break } } diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 7a025f531..20e995f7e 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -65,7 +65,7 @@ type AppConfig struct { // Expected version string for the sda-cli client (e.g., "v1.2.3") // If the client version header does not match this, the request is blocked. // Optional. Default value is "v0.0.0" - ExpectedCliVersion *semver.Version + ExpectedCliVersion *semver.Version ExpectedCliVersionStr string // This is the original string from the config file } diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index 306999419..b87288b4b 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -86,7 +86,6 @@ func (ts *TestSuite) TestAppConfig() { viper.Set("app.middleware", "noexist") viper.Set("app.expectedcliversion", "v0.2.0") - c := &Map{} err = c.appConfig() assert.Error(ts.T(), err, "Error expected") From df527f28f5c2031483855b7d4a0f1d3786d29f94 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 21:03:01 +0200 Subject: [PATCH 140/184] fix: lint, unhandled-error, for sda-download --- sda-download/api/s3/s3_test.go | 2 +- sda-download/api/sda/sda_test.go | 6 +++--- sda-download/internal/config/config.go | 2 +- sda-download/internal/database/database.go | 4 ++-- sda-download/internal/storage/seekable_test.go | 6 +++--- sda-download/internal/storage/storage_test.go | 6 +++--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index 96b703707..ab9f11290 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -279,7 +279,7 @@ func (ts *S3TestSuite) TestParseParams() { router.ServeHTTP(w, httptest.NewRequest("GET", params.Path, nil)) response := w.Result() - response.Body.Close() + _ = response.Body.Close() assert.Equal(ts.T(), http.StatusAccepted, response.StatusCode, "Request failed") } diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index faf35b449..516bae40d 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -746,7 +746,7 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { privPEM := "-----BEGIN PRIVATE KEY-----\n" + base64.StdEncoding.EncodeToString(privdata) + "\n-----END PRIVATE KEY-----\n" _, err = keyfile.Write([]byte(privPEM)) assert.NoError(t, err, "Could not write private key") - keyfile.Close() + _ = keyfile.Close() certfile, err := os.CreateTemp("", "cert") assert.NoError(t, err, "Could not create temp file for cert") @@ -754,7 +754,7 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { pubPEM := "-----BEGIN CERTIFICATE-----\n" + base64.StdEncoding.EncodeToString(server.Certificate().Raw) + "\n-----END CERTIFICATE-----\n" _, err = certfile.Write([]byte(pubPEM)) assert.NoError(t, err, "Could not write public key") - certfile.Close() + _ = certfile.Close() // Configure Reencrypt to use fake server config.Config.Reencrypt.Host = serverdetails[0] @@ -808,7 +808,7 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { _, err = io.Copy(datafile, &bufferWriter) assert.NoError(t, err, "Could not write temporary file") - datafile.Close() + _ = datafile.Close() // Substitute mock functions database.CheckFilePermission = func(_ string) (string, error) { diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 20e995f7e..3d16d78dd 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -515,7 +515,7 @@ func GetC4GHKeys() ([32]byte, string, error) { if err != nil { return [32]byte{}, "", fmt.Errorf("error when reading private key: %v", err) } - keyFile.Close() + _ = keyFile.Close() public := keys.DerivePublicKey(private) pem := bytes.Buffer{} diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index 3318aeabc..a5b2c2131 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -114,7 +114,7 @@ func (dbs *SQLdb) checkAndReconnectIfNeeded() { for dbs.DB.Ping() != nil { log.Errorln("Database unreachable, reconnecting") - dbs.DB.Close() + _ = dbs.DB.Close() if time.Since(start) > dbReconnectTimeout { logFatalf("Could not reconnect to failed database in reasonable time, giving up") @@ -461,5 +461,5 @@ func (dbs *SQLdb) getFile(fileID string) (*FileDownload, error) { // Close terminates the connection to the database func (dbs *SQLdb) Close() { db := dbs.DB - db.Close() + _ = db.Close() } diff --git a/sda-download/internal/storage/seekable_test.go b/sda-download/internal/storage/seekable_test.go index 7e9211a27..83598a53e 100644 --- a/sda-download/internal/storage/seekable_test.go +++ b/sda-download/internal/storage/seekable_test.go @@ -47,7 +47,7 @@ func TestSeekableBackend(t *testing.T) { assert.Equal(t, len(writeData), written, "Did not write all writeData") } - writer.Close() + _ = writer.Close() reader, err := backend.NewFileReadSeeker(path) assert.Nil(t, err, "s3 NewFileReadSeeker failed when it should work") @@ -182,7 +182,7 @@ func TestS3SeekablePrefetchSize(t *testing.T) { assert.NotNil(t, writer, "Got a nil reader for writer from s3") assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - writer.Close() + _ = writer.Close() reader, err := backend.NewFileReadSeeker(path) assert.Nil(t, err, "s3 NewFileReadSeeker failed when it should work") @@ -224,7 +224,7 @@ func TestS3SeekableSpecial(t *testing.T) { assert.Equal(t, len(writeData), written, "Did not write all writeData") } - writer.Close() + _ = writer.Close() reader, err := backend.NewFileReadSeeker(path) reader.(*s3Reader).seeked = true diff --git a/sda-download/internal/storage/storage_test.go b/sda-download/internal/storage/storage_test.go index 6bd0921a3..70352eb61 100644 --- a/sda-download/internal/storage/storage_test.go +++ b/sda-download/internal/storage/storage_test.go @@ -74,7 +74,7 @@ func writeName() (name string, err error) { func doCleanup() { for _, name := range cleanupFiles { - os.Remove(name) + _ = os.Remove(name) } cleanupFiles = cleanupFilesBack[0:0] @@ -138,7 +138,7 @@ func TestPosixBackend(t *testing.T) { assert.Nil(t, err, "Failure when writing to posix writer") assert.Equal(t, len(writeData), written, "Did not write all writeData") - writer.Close() + _ = writer.Close() log.SetOutput(&buf) writer, err = backend.NewFileWriter(posixNotCreatable) @@ -309,7 +309,7 @@ func TestS3Backend(t *testing.T) { assert.Nil(t, err, "Failure when writing to s3 writer") assert.Equal(t, len(writeData), written, "Did not write all writeData") - writer.Close() + _ = writer.Close() reader, err := s3back.NewFileReader(s3Creatable) assert.Nil(t, err, "s3 NewFileReader failed when it should work") From d0a63b39164dd67e4fe7af0507cd9c0fd6bbb839 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Oct 2025 21:11:34 +0200 Subject: [PATCH 141/184] fix: correctly sanitize userID in removeUserIDPrefix --- sda-download/internal/database/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index a5b2c2131..8e3999992 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -149,9 +149,9 @@ var GetFiles = func(datasetID string) ([]*FileInfo, error) { // removeUserIDPrefix strips the user id prefix from a file path func removeUserIDPrefix(filePath, userID string) string { - strings.ReplaceAll(userID, "@", "_") + sanitizedUserID := strings.ReplaceAll(userID, "@", "_") // Construct the full prefix we expect to find (userID + "/"). - fullPrefix := userID + "/" + fullPrefix := sanitizedUserID + "/" if strings.HasPrefix(filePath, fullPrefix) { return strings.TrimPrefix(filePath, fullPrefix) } From b60e9b7930b1072bdd09786de01a31e9543798f0 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Oct 2025 10:39:06 +0100 Subject: [PATCH 142/184] refactor: rename expectedcliversion to minimalcliversion --- sda-download/api/middleware/middleware.go | 8 +++---- .../api/middleware/middleware_test.go | 24 +++++++++---------- sda-download/dev_utils/config-notls.yaml | 2 +- sda-download/dev_utils/config.yaml | 2 +- sda-download/internal/config/config.go | 16 ++++++------- sda-download/internal/config/config_test.go | 16 ++++++------- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 7040d32d3..c16881eb1 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -115,7 +115,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { errorMessage := fmt.Sprintf( "Error: Your sda-cli client version '%s' is invalid. Required minimum version is '%s'.", clientVersionStr, - config.Config.App.ExpectedCliVersionStr, + config.Config.App.MinimalCliVersionStr, ) c.String(http.StatusPreconditionFailed, errorMessage) c.AbortWithStatus(http.StatusPreconditionFailed) @@ -124,13 +124,13 @@ func ClientVersionMiddleware() gin.HandlerFunc { } // 2. Check if the client version is sufficient (clientVersion >= requiredVersion) - if clientVersion.LessThan(config.Config.App.ExpectedCliVersion) { + if clientVersion.LessThan(config.Config.App.MinimalCliVersion) { errorMessage := fmt.Sprintf( "Error: Your sda-cli client version '%s' is insufficient. Please update to at least version '%s' to proceed.", clientVersionStr, - config.Config.App.ExpectedCliVersionStr, + config.Config.App.MinimalCliVersionStr, ) - log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.ExpectedCliVersionStr) + log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) c.String(http.StatusPreconditionFailed, errorMessage) c.AbortWithStatus(http.StatusPreconditionFailed) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 7c79b4f19..9e168daa9 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -307,9 +307,9 @@ func TestGetDatasets(t *testing.T) { } func TestClientVersionMiddleware(t *testing.T) { - originalExpectedCliVersion := config.Config.App.ExpectedCliVersion + originalMinimalCliVersion := config.Config.App.MinimalCliVersion defer func() { - config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + config.Config.App.MinimalCliVersion = originalMinimalCliVersion }() tests := []struct { @@ -363,13 +363,13 @@ func TestClientVersionMiddleware(t *testing.T) { r := httptest.NewRequest("GET", "/", nil) _, router := gin.CreateTestContext(w) - config.Config.App.ExpectedCliVersionStr = tt.configExpectedVersion + config.Config.App.MinimalCliVersionStr = tt.configExpectedVersion // Set the configuration mock by parsing the string into the required SemVer object parsedVersion, err := semver.NewVersion(tt.configExpectedVersion) if err != nil { t.Fatalf("Test setup error: Failed to parse expected version '%s': %v", tt.configExpectedVersion, err) } - config.Config.App.ExpectedCliVersion = parsedVersion + config.Config.App.MinimalCliVersion = parsedVersion if tt.clientVersionHeader != "" { r.Header.Set("SDA-Client-Version", tt.clientVersionHeader) @@ -431,14 +431,14 @@ func TestChainDefaultMiddleware_Success(t *testing.T) { }() // Setup config for ClientVersionMiddleware Success - originalExpectedCliVersion := config.Config.App.ExpectedCliVersion - expectedCliVersion, err := semver.NewVersion("0.2.0") + originalMinimalCliVersion := config.Config.App.MinimalCliVersion + parsedVersion, err := semver.NewVersion("0.2.0") if err != nil { t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) } - config.Config.App.ExpectedCliVersion = expectedCliVersion + config.Config.App.MinimalCliVersion = parsedVersion defer func() { - config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + config.Config.App.MinimalCliVersion = originalMinimalCliVersion }() // Setup Request/Response @@ -486,14 +486,14 @@ func TestChainDefaultMiddleware_Fail_VersionAbortsChain(t *testing.T) { }() // Setup config for ClientVersionMiddleware Failure (Insufficient version) - originalExpectedCliVersion := config.Config.App.ExpectedCliVersion - expectedCliVersion, err := semver.NewVersion("0.2.0") + originalMinimalCliVersion := config.Config.App.MinimalCliVersion + minimalCliVersion, err := semver.NewVersion("0.2.0") if err != nil { t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) } - config.Config.App.ExpectedCliVersion = expectedCliVersion + config.Config.App.MinimalCliVersion = minimalCliVersion defer func() { - config.Config.App.ExpectedCliVersion = originalExpectedCliVersion + config.Config.App.MinimalCliVersion = originalMinimalCliVersion }() // Setup Request/Response diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 95e5117d8..4752b5488 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -1,5 +1,5 @@ app: - expectedcliversion: "v0.2.0" + minimalcliversion: "v0.2.0" log: level: "debug" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index f581487e8..91bb62cc5 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -4,7 +4,7 @@ app: serverkey: "./dev_utils/certs/download-key.pem" port: "8443" middleware: "default" - expectedcliversion: "v0.2.0" + minimalcliversion: "v0.2.0" log: level: "debug" diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 3d16d78dd..f71664406 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -62,11 +62,11 @@ type AppConfig struct { // Optional. Default value is "default" for TokenMiddleware Middleware string - // Expected version string for the sda-cli client (e.g., "v1.2.3") + // Minimal version string for the sda-cli client (e.g., "v1.2.3") // If the client version header does not match this, the request is blocked. // Optional. Default value is "v0.0.0" - ExpectedCliVersion *semver.Version - ExpectedCliVersionStr string // This is the original string from the config file + MinimalCliVersion *semver.Version + MinimalCliVersionStr string // This is the original string from the config file } // Stores the Crypt4GH private key used internally @@ -249,7 +249,7 @@ func (c *Map) applyDefaults() { viper.SetDefault("app.host", "0.0.0.0") viper.SetDefault("app.port", 8080) viper.SetDefault("app.middleware", "default") - viper.SetDefault("app.expectedcliversion", "v0.0.0") + viper.SetDefault("app.minimalcliversion", "v0.0.0") viper.SetDefault("session.expiration", -1) viper.SetDefault("session.secure", true) viper.SetDefault("session.httponly", true) @@ -380,12 +380,12 @@ func (c *Map) appConfig() error { c.App.Middleware = viper.GetString("app.middleware") // Validate and parse the configured minimum client version into a SemVer object - c.App.ExpectedCliVersionStr = viper.GetString("app.expectedcliversion") - parsedVersion, err := semver.NewVersion(c.App.ExpectedCliVersionStr) + c.App.MinimalCliVersionStr = viper.GetString("app.minimalcliversion") + parsedVersion, err := semver.NewVersion(c.App.MinimalCliVersionStr) if err != nil { - return fmt.Errorf("app.expectedcliversion value='%s' is not a valid semantic version: %v", c.App.ExpectedCliVersionStr, err) + return fmt.Errorf("app.minimalcliversion value='%s' is not a valid semantic version: %v", c.App.MinimalCliVersionStr, err) } - c.App.ExpectedCliVersion = parsedVersion + c.App.MinimalCliVersion = parsedVersion if c.App.Port != 443 && c.App.Port != 8080 { c.App.Port = viper.GetInt("app.port") diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index b87288b4b..e06308ec0 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -84,15 +84,15 @@ func (ts *TestSuite) TestAppConfig() { viper.Set("db.sslmode", "disable") viper.Set("app.middleware", "noexist") - viper.Set("app.expectedcliversion", "v0.2.0") + viper.Set("app.minimalcliversion", "v0.2.0") c := &Map{} err = c.appConfig() assert.Error(ts.T(), err, "Error expected") viper.Reset() - // Test fail on invalid expected client version - viper.Set("app.expectedcliversion", "not-a-semver") + // Test fail on invalid minimal client version + viper.Set("app.minimalcliversion", "not-a-semver") c = &Map{} err = c.appConfig() assert.Error(ts.T(), err, "Error expected for invalid semver string") @@ -103,7 +103,7 @@ func (ts *TestSuite) TestAppConfig() { viper.Set("app.port", 1234) viper.Set("app.servercert", "test") viper.Set("app.serverkey", "test") - viper.Set("app.expectedcliversion", "v0.2.0") + viper.Set("app.minimalcliversion", "v0.2.0") viper.Set("log.logLevel", "debug") viper.Set("db.sslmode", "disable") viper.Set("c4gh.transientKeyPath", privateKeyFile.Name()) @@ -119,10 +119,10 @@ func (ts *TestSuite) TestAppConfig() { assert.NotEmpty(ts.T(), c.C4GH.PrivateKey) assert.NotEmpty(ts.T(), c.C4GH.PublicKeyB64) - // Assert ExpectedCliVersion - expectedVersion, _ := semver.NewVersion("v0.2.0") - assert.NotNil(ts.T(), c.App.ExpectedCliVersion, "ExpectedCliVersion should be parsed and not nil") - assert.True(ts.T(), expectedVersion.Equal(c.App.ExpectedCliVersion), "Parsed version does not match expected version") + // Assert MinimalCliVersion + parsedVersion, _ := semver.NewVersion("v0.2.0") + assert.NotNil(ts.T(), c.App.MinimalCliVersion, "MinimalCliVersion should be parsed and not nil") + assert.True(ts.T(), parsedVersion.Equal(c.App.MinimalCliVersion), "Parsed version does not match minimal version") // Check the private key that was loaded by checking the derived public key publicKey, err := base64.StdEncoding.DecodeString(c.C4GH.PublicKeyB64) From cc8e54b1c2d8da0f4e097fb5076b4716727f8d18 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Oct 2025 11:02:58 +0100 Subject: [PATCH 143/184] feat: check token before cli version in the middleware --- sda-download/api/middleware/middleware.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index c16881eb1..51c6f1ab5 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -81,9 +81,6 @@ func TokenMiddleware() gin.HandlerFunc { // Store dataset list to request context, for use in the endpoint handlers log.Debugf("storing %v to request context", cache) c.Set(requestContextKey, cache) - - // Forward request to the next endpoint handler - c.Next() } } @@ -142,20 +139,26 @@ func ClientVersionMiddleware() gin.HandlerFunc { } } -// ChainDefaultMiddleware chains the ClientVersionMiddleware and TokenMiddleware. +// ChainDefaultMiddleware chains the TokenMiddleware and ClientVersionMiddleware, +// prioritizing authentication before checking client version requirements. // It is intended to be the default composite middleware set for the application. func ChainDefaultMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - // 1. Run the Client Version Check. This will abort if version is invalid/missing (HTTP 412) + // Run the Token Middleware. This will abort if authentication fails (HTTP 401) + TokenMiddleware()(c) + + if c.IsAborted() { + return + } + + // Run the Client Version Check (only if token check passed). This will abort if version is invalid/missing (HTTP 412) ClientVersionMiddleware()(c) - // Check if the request was aborted by the version middleware if c.IsAborted() { return } - // 2. Run the Token Middleware (only if version check passed) - TokenMiddleware()(c) + c.Next() } } From 84b7cdc0a6685ca29f4b00fb1dc04230987ae061 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Oct 2025 16:27:27 +0100 Subject: [PATCH 144/184] fix: integration test after changing the order of check --- .../tests/common/50_check_endpoint.sh | 81 +++++++++---------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/sda-download/.github/integration/tests/common/50_check_endpoint.sh b/sda-download/.github/integration/tests/common/50_check_endpoint.sh index 91b9fb07a..eb10a9ac3 100755 --- a/sda-download/.github/integration/tests/common/50_check_endpoint.sh +++ b/sda-download/.github/integration/tests/common/50_check_endpoint.sh @@ -30,12 +30,38 @@ fi echo "Head method health endpoint is ok" # ------------------ +# Test empty token + +check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) + +if [ "$check_401" != "401" ]; then + echo "no token provided should give 401" + echo "got: ${check_401}" + exit 1 +fi + +echo "got correct response when no token provided" + +check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) + +if [ "$check_405" != "405" ]; then + echo "POST should not be allowed" + echo "got: ${check_405}" + exit 1 +fi + +echo "got correct response when POST method used" + +# ------------------ +# Test good token + +token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') + # Test Client Version Header -# These tests verify that the version middleware runs before authentication. # We assume the app is configured to require a minimum version (v0.2.0). # Fail - missing header (expected 412 Precondition Failed) -check_missing_header=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem "https://localhost:8443/metadata/datasets") +check_missing_header=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets") if [ "$check_missing_header" != "412" ]; then echo "Client Version Test FAIL: missing header should return 412" @@ -45,7 +71,7 @@ fi echo "Client Version Test OK: Missing header correctly returns 412" # Fail - insufficient version (e.g., v0.1.0, Expected 412) -check_insufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.1.0" "https://localhost:8443/metadata/datasets") +check_insufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.1.0" "https://localhost:8443/metadata/datasets") if [ "$check_insufficient_version" != "412" ]; then echo "Client Version Test FAIL: insufficient version (v0.1.0) should return 412" @@ -54,54 +80,25 @@ if [ "$check_insufficient_version" != "412" ]; then fi echo "Client Version Test OK: Insufficient version (v0.1.0) correctly returns 412" +# Success - sufficient version (e.g., v0.2.0, Expected 200) +check_sufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.2.0" "https://localhost:8443/metadata/datasets") -# Success - sufficient version (e.g., v0.2.0, Expected 401 from token middleware) -check_sufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.2.0" "https://localhost:8443/metadata/datasets") - -if [ "$check_sufficient_version" != "401" ]; then - echo "Client Version Test FAIL: sufficient version (v0.2.0) passed version check but failed token check (Expected 401)" +if [ "$check_sufficient_version" != "200" ]; then + echo "Client Version Test FAIL: sufficient version (v0.2.0) should pass version check and return 200" echo "got: ${check_sufficient_version}" exit 1 fi -echo "Client Version Test OK: sufficient version (v0.2.0) correctly proceeds to token check (returns 401)" +echo "Client Version Test OK: sufficient version (v0.2.0) correctly returns 200" -# Test 4: Success - newer version (e.g., v1.0.0, Expected 401 from token middleware) -check_newer_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v1.0.0" "https://localhost:8443/metadata/datasets") +# Success - newer version (e.g., v1.0.0, Expected 200) +check_newer_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v1.0.0" "https://localhost:8443/metadata/datasets") -if [ "$check_newer_version" != "401" ]; then - echo "Client Version Test FAIL: Newer version (v1.0.0) passed version check but failed token check (Expected 401)" +if [ "$check_newer_version" != "200" ]; then + echo "Client Version Test FAIL: Newer version (v1.0.0) should pass version check and return 200" echo "got: ${check_newer_version}" exit 1 fi -echo "Client Version Test OK: Newer version (v1.0.0) correctly proceeds to token check (returns 401)" - -# ------------------ -# Test empty token - -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) - -if [ "$check_401" != "401" ]; then - echo "no token provided should give 401" - echo "got: ${check_401}" - exit 1 -fi - -echo "got correct response when no token provided" - -check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) - -if [ "$check_405" != "405" ]; then - echo "POST should not be allowed" - echo "got: ${check_405}" - exit 1 -fi - -echo "got correct response when POST method used" - -# ------------------ -# Test good token - -token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') +echo "Client Version Test OK: Newer version (v1.0.0) correctly returns 200" ## Test datasets endpoint From 66f0f065b13237f15cfa25f0f7e416616347cb91 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Oct 2025 17:37:10 +0100 Subject: [PATCH 145/184] refactor: merge unit tests for ChainDefaultMiddleware --- .../api/middleware/middleware_test.go | 245 ++++++++++-------- 1 file changed, 130 insertions(+), 115 deletions(-) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 9e168daa9..bc928888f 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -3,6 +3,7 @@ package middleware import ( "bytes" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -16,6 +17,7 @@ import ( "github.com/neicnordic/sda-download/internal/session" "github.com/neicnordic/sda-download/pkg/auth" log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" ) const token string = "token" @@ -313,46 +315,46 @@ func TestClientVersionMiddleware(t *testing.T) { }() tests := []struct { - name string - clientVersionHeader string - configExpectedVersion string - expectedStatus int - expectedBodyContains string + name string + clientVersionHeader string + configMinimalVersion string + expectedStatus int + expectedBodyContains string }{ { - name: "Fail_MissingHeader", - clientVersionHeader: "", - configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "Missing required header", + name: "Fail_MissingHeader", + clientVersionHeader: "", + configMinimalVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "Missing required header", }, { - name: "Fail_InvalidClientSemVer", - clientVersionHeader: "v-invalid-1", - configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is invalid", + name: "Fail_InvalidClientSemVer", + clientVersionHeader: "v-invalid-1", + configMinimalVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is invalid", }, { - name: "Fail_InsufficientVersion", - clientVersionHeader: "v0.1.9", - configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", + name: "Fail_InsufficientVersion", + clientVersionHeader: "v0.1.9", + configMinimalVersion: "v0.2.0", + expectedStatus: http.StatusPreconditionFailed, // 412 + expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", }, { - name: "Success_EqualVersion", - clientVersionHeader: "v0.2.0", - configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusOK, // 200 - expectedBodyContains: "", + name: "Success_EqualVersion", + clientVersionHeader: "v0.2.0", + configMinimalVersion: "v0.2.0", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", }, { - name: "Success_NewerVersion", - clientVersionHeader: "v0.3.0", - configExpectedVersion: "v0.2.0", - expectedStatus: http.StatusOK, // 200 - expectedBodyContains: "", + name: "Success_NewerVersion", + clientVersionHeader: "v0.3.0", + configMinimalVersion: "v0.2.0", + expectedStatus: http.StatusOK, // 200 + expectedBodyContains: "", }, } @@ -363,11 +365,11 @@ func TestClientVersionMiddleware(t *testing.T) { r := httptest.NewRequest("GET", "/", nil) _, router := gin.CreateTestContext(w) - config.Config.App.MinimalCliVersionStr = tt.configExpectedVersion + config.Config.App.MinimalCliVersionStr = tt.configMinimalVersion // Set the configuration mock by parsing the string into the required SemVer object - parsedVersion, err := semver.NewVersion(tt.configExpectedVersion) + parsedVersion, err := semver.NewVersion(tt.configMinimalVersion) if err != nil { - t.Fatalf("Test setup error: Failed to parse expected version '%s': %v", tt.configExpectedVersion, err) + t.Fatalf("Test setup error: Failed to parse minimal version '%s': %v", tt.configMinimalVersion, err) } config.Config.App.MinimalCliVersion = parsedVersion @@ -408,111 +410,124 @@ func TestClientVersionMiddleware(t *testing.T) { } } -func TestChainDefaultMiddleware_Success(t *testing.T) { - // Setup global mocks required for TokenMiddleware Success (No Cache) +// TestChainDefaultMiddleware tests the ordered execution of TokenMiddleware -> ClientVersionMiddleware +func TestChainDefaultMiddleware(t *testing.T) { + // Store original configuration and functions for restoration originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas originalGetPermissions := auth.GetPermissions originalNewSessionKey := session.NewSessionKey originalSessionName := config.Config.Session.Name + originalMinimalCliVersion := config.Config.App.MinimalCliVersion - auth.GetToken = func(_ http.Header) (string, int, error) { return token, 200, nil } - auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { return &auth.Visas{}, nil } - auth.GetPermissions = func(_ auth.Visas) []string { return []string{"dataset1"} } - session.NewSessionKey = func() string { return "key" } - config.Config.Session.Name = "sda_session_key" // Set session name for cookie assertion - + // Shared defer to restore mocks and config after all tests run defer func() { auth.GetToken = originalGetToken auth.GetVisas = originalGetVisas auth.GetPermissions = originalGetPermissions session.NewSessionKey = originalNewSessionKey config.Config.Session.Name = originalSessionName - }() - - // Setup config for ClientVersionMiddleware Success - originalMinimalCliVersion := config.Config.App.MinimalCliVersion - parsedVersion, err := semver.NewVersion("0.2.0") - if err != nil { - t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) - } - config.Config.App.MinimalCliVersion = parsedVersion - defer func() { config.Config.App.MinimalCliVersion = originalMinimalCliVersion }() - // Setup Request/Response - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/", nil) - r.Header.Set("SDA-Client-Version", "v0.3.0") // Newer version, should pass - _, router := gin.CreateTestContext(w) - - // Send request through the chain middleware - router.GET("/", ChainDefaultMiddleware(), testEndpoint) - router.ServeHTTP(w, r) - - // Assertions - expectedStatusCode := http.StatusOK // Both middlewares passed - if w.Code != expectedStatusCode { - t.Errorf("TestChainDefaultMiddleware_Success failed, got status %d expected %d", w.Code, expectedStatusCode) - } - // Check that a session cookie was set by TokenMiddleware (confirming it ran) - cookies := w.Result().Cookies() - cookieFound := false - for _, c := range cookies { - if c.Name == "sda_session_key" { - cookieFound = true - - break - } + // Define test cases for the middleware chain + tests := []struct { + name string + clientVersion string + minimalVersion string + mockTokenSuccess bool + expectedStatus int + expectCookie bool + }{ + { + name: "Success_TokenAndVersionPass", + clientVersion: "v0.3.0", + minimalVersion: "v0.2.0", + mockTokenSuccess: true, + expectedStatus: http.StatusOK, + expectCookie: true, + }, + { + name: "Fail_VersionInsufficient_AfterTokenPass", + clientVersion: "v0.1.0", // Fails version check + minimalVersion: "v0.2.0", + mockTokenSuccess: true, + expectedStatus: http.StatusPreconditionFailed, // 412 + expectCookie: true, // TokenMiddleware ran and set cookie + }, + { + name: "Fail_AuthBlockedFirst_VersionIrrelevant", + clientVersion: "v0.1.0", // Would fail version check, but should fail token first + minimalVersion: "v0.2.0", + mockTokenSuccess: false, + expectedStatus: http.StatusUnauthorized, // 401 + expectCookie: false, + }, + { + name: "Fail_AuthBlockedFirst_VersionSufficient", + clientVersion: "v1.0.0", // Would pass version check, but should fail token first + minimalVersion: "v0.2.0", + mockTokenSuccess: false, + expectedStatus: http.StatusUnauthorized, // 401 + expectCookie: false, + }, } - if !cookieFound { - t.Error("TestChainDefaultMiddleware_Success failed, expected a session cookie, but none was found.") - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // --- Setup Mocks and Config for this test case --- + token := "mock-token" + + if tt.mockTokenSuccess { + auth.GetToken = func(_ http.Header) (string, int, error) { return token, http.StatusOK, nil } + auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { return &auth.Visas{}, nil } + auth.GetPermissions = func(_ auth.Visas) []string { return []string{"dataset1"} } + session.NewSessionKey = func() string { return "key" } + config.Config.Session.Name = "sda_session_key" + } else { + auth.GetToken = func(_ http.Header) (string, int, error) { + return "", http.StatusUnauthorized, errors.New("missing token") + } + auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { + return &auth.Visas{}, errors.New("auth failed") + } + } -func TestChainDefaultMiddleware_Fail_VersionAbortsChain(t *testing.T) { - // We use a flag to assert that TokenMiddleware was NOT executed. - originalGetToken := auth.GetToken - wasTokenCalled := false - auth.GetToken = func(_ http.Header) (string, int, error) { - wasTokenCalled = true + // Set up ClientVersionMiddleware config + parsedVersion, err := semver.NewVersion(tt.minimalVersion) + assert.NoErrorf(t, err, "Test setup error: Failed to parse minimal version '%s'", tt.minimalVersion) + config.Config.App.MinimalCliVersion = parsedVersion - return token, 200, nil - } - defer func() { - auth.GetToken = originalGetToken - }() + // --- Execution --- - // Setup config for ClientVersionMiddleware Failure (Insufficient version) - originalMinimalCliVersion := config.Config.App.MinimalCliVersion - minimalCliVersion, err := semver.NewVersion("0.2.0") - if err != nil { - t.Fatalf("Test setup error: Failed to parse expected version '0.2.0': %v", err) - } - config.Config.App.MinimalCliVersion = minimalCliVersion - defer func() { - config.Config.App.MinimalCliVersion = originalMinimalCliVersion - }() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", "/", nil) + if tt.clientVersion != "" { + r.Header.Set("SDA-Client-Version", tt.clientVersion) + } - // Setup Request/Response - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/", nil) - r.Header.Set("SDA-Client-Version", "0.1.0") // Insufficient version, should abort (412) - _, router := gin.CreateTestContext(w) + _, router := gin.CreateTestContext(w) + router.GET("/", ChainDefaultMiddleware(), testEndpoint) + router.ServeHTTP(w, r) - // Send request through the chain middleware - router.GET("/", ChainDefaultMiddleware(), testEndpoint) - router.ServeHTTP(w, r) + // Assert Status Code + assert.Equal(t, tt.expectedStatus, w.Code, fmt.Sprintf("Expected status %d, got %d", tt.expectedStatus, w.Code)) - // Assertions - expectedStatusCode := http.StatusPreconditionFailed // 412 - if w.Code != expectedStatusCode { - t.Errorf("TestChainDefaultMiddleware_Fail_VersionAbortsChain failed, got status %d expected %d", w.Code, expectedStatusCode) - } + // Assert Cookie Presence + cookieFound := false + for _, c := range w.Result().Cookies() { + if c.Name == config.Config.Session.Name { + cookieFound = true - if wasTokenCalled { - t.Error("TestChainDefaultMiddleware_Fail_VersionAbortsChain failed, TokenMiddleware was executed when it should have been aborted.") + break + } + } + + if tt.expectCookie { + assert.True(t, cookieFound, "Expected a session cookie to be set, but none was found.") + } else { + assert.False(t, cookieFound, "Expected no session cookie to be set, but one was found.") + } + }) } } From cecbe91b2896cb372bf1bb8c9b965cf2e343c4da Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 28 Oct 2025 13:40:00 +0100 Subject: [PATCH 146/184] feat: remove duplicated logic to return http code --- sda-download/api/middleware/middleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 51c6f1ab5..6238a24c8 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -100,7 +100,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { ) log.Warnf("request blocked (412): Missing required header '%s'", headerName) c.String(http.StatusPreconditionFailed, errorMessage) - c.AbortWithStatus(http.StatusPreconditionFailed) + c.Abort() return } @@ -115,7 +115,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { config.Config.App.MinimalCliVersionStr, ) c.String(http.StatusPreconditionFailed, errorMessage) - c.AbortWithStatus(http.StatusPreconditionFailed) + c.Abort() return } @@ -129,7 +129,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { ) log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) c.String(http.StatusPreconditionFailed, errorMessage) - c.AbortWithStatus(http.StatusPreconditionFailed) + c.Abort() return } From 2e5b9674d7020e9b3d6241813d3bbb41069013c0 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 28 Oct 2025 14:25:01 +0100 Subject: [PATCH 147/184] refactor: use a more idiomatic approach for the middleware --- sda-download/api/api.go | 32 ++++++++++++------- sda-download/api/middleware/middleware.go | 32 +++++++------------ .../api/middleware/middleware_test.go | 2 +- sda-download/cmd/main.go | 7 ---- sda-download/internal/config/config.go | 2 +- 5 files changed, 35 insertions(+), 40 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index ae1a4c8dd..2ccf5de31 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -9,18 +9,24 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/neicnordic/sda-download/api/middleware" "github.com/neicnordic/sda-download/api/s3" "github.com/neicnordic/sda-download/api/sda" "github.com/neicnordic/sda-download/internal/config" log "github.com/sirupsen/logrus" ) -// SelectedMiddleware is used to control authentication and authorization -// behaviour with config app.middleware -// available middlewares: -// "default" for TokenMiddleware -var SelectedMiddleware = func() gin.HandlerFunc { - return nil +// SelectedMiddleware returns the middleware chain based on configuration. +// For example, config.Config.App.Middleware could be "default", "token", etc. +var SelectedMiddleware = func() []gin.HandlerFunc { + switch strings.ToLower(config.Config.App.Middleware) { + case "default": + return middleware.ChainDefaultMiddleware() + case "token": + return []gin.HandlerFunc{middleware.TokenMiddleware()} + default: + return nil + } } // healthResponse @@ -65,12 +71,16 @@ func Setup() *http.Server { } router.HandleMethodNotAllowed = true + mw := SelectedMiddleware() + + // protected endpoints + router.GET("/metadata/datasets", append(mw, sda.Datasets)...) + router.GET("/metadata/datasets/*dataset", append(mw, sda.Files)...) + router.GET("/files/:fileid", append(mw, sda.Download)...) + router.GET("/s3/*path", append(mw, s3.Download)...) + router.HEAD("/s3/*path", append(mw, s3.Download)...) - router.GET("/metadata/datasets", SelectedMiddleware(), sda.Datasets) - router.GET("/metadata/datasets/*dataset", SelectedMiddleware(), sda.Files) - router.GET("/files/:fileid", SelectedMiddleware(), sda.Download) - router.GET("/s3/*path", SelectedMiddleware(), s3.Download) - router.HEAD("/s3/*path", SelectedMiddleware(), s3.Download) + // public endpoints router.GET("/health", healthResponse) router.HEAD("/", healthResponse) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 6238a24c8..059b62ef7 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -81,6 +81,9 @@ func TokenMiddleware() gin.HandlerFunc { // Store dataset list to request context, for use in the endpoint handlers log.Debugf("storing %v to request context", cache) c.Set(requestContextKey, cache) + + // Forward request to the next endpoint handler + c.Next() } } @@ -136,29 +139,18 @@ func ClientVersionMiddleware() gin.HandlerFunc { // Version is correct, proceed to the next handler/middleware log.Debugf("client version check passed: %s", clientVersionStr) + + // Forward request to the next endpoint handler + c.Next() } } -// ChainDefaultMiddleware chains the TokenMiddleware and ClientVersionMiddleware, -// prioritizing authentication before checking client version requirements. -// It is intended to be the default composite middleware set for the application. -func ChainDefaultMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { - // Run the Token Middleware. This will abort if authentication fails (HTTP 401) - TokenMiddleware()(c) - - if c.IsAborted() { - return - } - - // Run the Client Version Check (only if token check passed). This will abort if version is invalid/missing (HTTP 412) - ClientVersionMiddleware()(c) - - if c.IsAborted() { - return - } - - c.Next() +// ChainDefaultMiddleware returns the default set of middlewares +// to be applied in order: authentication, then client version check. +func ChainDefaultMiddleware() []gin.HandlerFunc { + return []gin.HandlerFunc{ + TokenMiddleware(), + ClientVersionMiddleware(), } } diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index bc928888f..97873703e 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -507,7 +507,7 @@ func TestChainDefaultMiddleware(t *testing.T) { } _, router := gin.CreateTestContext(w) - router.GET("/", ChainDefaultMiddleware(), testEndpoint) + router.GET("/", append(ChainDefaultMiddleware(), testEndpoint)...) router.ServeHTTP(w, r) // Assert Status Code diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index d9113c541..2c55930b6 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -2,7 +2,6 @@ package main import ( "github.com/neicnordic/sda-download/api" - "github.com/neicnordic/sda-download/api/middleware" "github.com/neicnordic/sda-download/api/sda" "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/database" @@ -24,12 +23,6 @@ func init() { } config.Config = *conf - // Set middleware - // nolint:gocritic // this nolint can be removed, if you have more than one middlewares available - switch conf.App.Middleware { //nolint:revive - default: - api.SelectedMiddleware = middleware.ChainDefaultMiddleware - } log.Infof("%s middleware selected", conf.App.Middleware) // Connect to database diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index f71664406..cba7baeca 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -25,7 +25,7 @@ const S3 = "s3" // availableMiddlewares list the options for middlewares // empty string "" is an alias for default, for when the config key is not set, or it's empty -var availableMiddlewares = []string{"", "default"} +var availableMiddlewares = []string{"", "default", "token"} // Config is a global configuration value store var Config Map From 65e5d73ede4f58827be8aeed6cc3d813cd25cf36 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 3 Nov 2025 15:18:51 +0100 Subject: [PATCH 148/184] feat: apply reviewers comments Co-authored-by: Joakim Bygdell --- sda-download/api/api.go | 17 +-- sda-download/api/middleware/middleware.go | 41 ++---- .../api/middleware/middleware_test.go | 130 +----------------- 3 files changed, 21 insertions(+), 167 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 2ccf5de31..85bba3d9a 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -21,7 +21,10 @@ import ( var SelectedMiddleware = func() []gin.HandlerFunc { switch strings.ToLower(config.Config.App.Middleware) { case "default": - return middleware.ChainDefaultMiddleware() + return []gin.HandlerFunc{ + middleware.TokenMiddleware(), + middleware.ClientVersionMiddleware(), + } case "token": return []gin.HandlerFunc{middleware.TokenMiddleware()} default: @@ -71,14 +74,12 @@ func Setup() *http.Server { } router.HandleMethodNotAllowed = true - mw := SelectedMiddleware() - // protected endpoints - router.GET("/metadata/datasets", append(mw, sda.Datasets)...) - router.GET("/metadata/datasets/*dataset", append(mw, sda.Files)...) - router.GET("/files/:fileid", append(mw, sda.Download)...) - router.GET("/s3/*path", append(mw, s3.Download)...) - router.HEAD("/s3/*path", append(mw, s3.Download)...) + router.GET("/metadata/datasets", append(SelectedMiddleware(), sda.Datasets)...) + router.GET("/metadata/datasets/*dataset", append(SelectedMiddleware(), sda.Files)...) + router.GET("/files/:fileid", append(SelectedMiddleware(), sda.Download)...) + router.GET("/s3/*path", append(SelectedMiddleware(), s3.Download)...) + router.HEAD("/s3/*path", append(SelectedMiddleware(), s3.Download)...) // public endpoints router.GET("/health", healthResponse) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 059b62ef7..6f5e4fb77 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -92,17 +92,12 @@ func TokenMiddleware() gin.HandlerFunc { // if the version does not meet the minimum required version. func ClientVersionMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - const headerName = "SDA-Client-Version" - clientVersionStr := c.GetHeader(headerName) + clientVersionStr := c.GetHeader("SDA-Client-Version") - // 1. Check if the header is present + // Check if the header is present if clientVersionStr == "" { - errorMessage := fmt.Sprintf( - "Error: Missing required header '%s'. Please ensure you are using the latest sda-cli client.", - headerName, - ) - log.Warnf("request blocked (412): Missing required header '%s'", headerName) - c.String(http.StatusPreconditionFailed, errorMessage) + log.Warnf("request blocked (412): Missing client version header in request") + c.String(http.StatusPreconditionFailed, "Missing client version header in request") c.Abort() return @@ -111,26 +106,17 @@ func ClientVersionMiddleware() gin.HandlerFunc { // Parse the client's provided version (using the processed string) clientVersion, err := semver.NewVersion(clientVersionStr) if err != nil { - log.Warnf("request blocked (412): processed client version header '%s' is not a valid semantic version: %v", clientVersionStr, err) - errorMessage := fmt.Sprintf( - "Error: Your sda-cli client version '%s' is invalid. Required minimum version is '%s'.", - clientVersionStr, - config.Config.App.MinimalCliVersionStr, - ) - c.String(http.StatusPreconditionFailed, errorMessage) + log.Warnf("client version header '%s' is not a valid semantic version: %v", clientVersionStr, err) + c.String(http.StatusPreconditionFailed, "client version header is not a valid semantic version") c.Abort() return } - // 2. Check if the client version is sufficient (clientVersion >= requiredVersion) + // Check if the client version is sufficient (clientVersion >= minimalVersion) if clientVersion.LessThan(config.Config.App.MinimalCliVersion) { - errorMessage := fmt.Sprintf( - "Error: Your sda-cli client version '%s' is insufficient. Please update to at least version '%s' to proceed.", - clientVersionStr, - config.Config.App.MinimalCliVersionStr, - ) - log.Warnf("request blocked (412): Insufficient client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) + errorMessage := fmt.Sprintf("Error: Your sda-cli client version is outdated, please update to at least version '%s'.", config.Config.App.MinimalCliVersionStr) + log.Warnf("request blocked (412): outdated client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) c.String(http.StatusPreconditionFailed, errorMessage) c.Abort() @@ -145,15 +131,6 @@ func ClientVersionMiddleware() gin.HandlerFunc { } } -// ChainDefaultMiddleware returns the default set of middlewares -// to be applied in order: authentication, then client version check. -func ChainDefaultMiddleware() []gin.HandlerFunc { - return []gin.HandlerFunc{ - TokenMiddleware(), - ClientVersionMiddleware(), - } -} - // GetCacheFromContext is a helper function that endpoints can use to get data // stored to the *current* request context (not the session storage). // The request context was populated by the middleware, which in turn uses the session storage. diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 97873703e..ffdcd35c0 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -3,7 +3,6 @@ package middleware import ( "bytes" "errors" - "fmt" "io" "net/http" "net/http/httptest" @@ -17,7 +16,6 @@ import ( "github.com/neicnordic/sda-download/internal/session" "github.com/neicnordic/sda-download/pkg/auth" log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" ) const token string = "token" @@ -326,21 +324,21 @@ func TestClientVersionMiddleware(t *testing.T) { clientVersionHeader: "", configMinimalVersion: "v0.2.0", expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "Missing required header", + expectedBodyContains: "Missing client version header in request", }, { name: "Fail_InvalidClientSemVer", clientVersionHeader: "v-invalid-1", configMinimalVersion: "v0.2.0", expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is invalid", + expectedBodyContains: "is not a valid semantic version", }, { name: "Fail_InsufficientVersion", clientVersionHeader: "v0.1.9", configMinimalVersion: "v0.2.0", expectedStatus: http.StatusPreconditionFailed, // 412 - expectedBodyContains: "is insufficient. Please update to at least version 'v0.2.0'", + expectedBodyContains: "is outdated, please update to at least version 'v0.2.0'", }, { name: "Success_EqualVersion", @@ -409,125 +407,3 @@ func TestClientVersionMiddleware(t *testing.T) { }) } } - -// TestChainDefaultMiddleware tests the ordered execution of TokenMiddleware -> ClientVersionMiddleware -func TestChainDefaultMiddleware(t *testing.T) { - // Store original configuration and functions for restoration - originalGetToken := auth.GetToken - originalGetVisas := auth.GetVisas - originalGetPermissions := auth.GetPermissions - originalNewSessionKey := session.NewSessionKey - originalSessionName := config.Config.Session.Name - originalMinimalCliVersion := config.Config.App.MinimalCliVersion - - // Shared defer to restore mocks and config after all tests run - defer func() { - auth.GetToken = originalGetToken - auth.GetVisas = originalGetVisas - auth.GetPermissions = originalGetPermissions - session.NewSessionKey = originalNewSessionKey - config.Config.Session.Name = originalSessionName - config.Config.App.MinimalCliVersion = originalMinimalCliVersion - }() - - // Define test cases for the middleware chain - tests := []struct { - name string - clientVersion string - minimalVersion string - mockTokenSuccess bool - expectedStatus int - expectCookie bool - }{ - { - name: "Success_TokenAndVersionPass", - clientVersion: "v0.3.0", - minimalVersion: "v0.2.0", - mockTokenSuccess: true, - expectedStatus: http.StatusOK, - expectCookie: true, - }, - { - name: "Fail_VersionInsufficient_AfterTokenPass", - clientVersion: "v0.1.0", // Fails version check - minimalVersion: "v0.2.0", - mockTokenSuccess: true, - expectedStatus: http.StatusPreconditionFailed, // 412 - expectCookie: true, // TokenMiddleware ran and set cookie - }, - { - name: "Fail_AuthBlockedFirst_VersionIrrelevant", - clientVersion: "v0.1.0", // Would fail version check, but should fail token first - minimalVersion: "v0.2.0", - mockTokenSuccess: false, - expectedStatus: http.StatusUnauthorized, // 401 - expectCookie: false, - }, - { - name: "Fail_AuthBlockedFirst_VersionSufficient", - clientVersion: "v1.0.0", // Would pass version check, but should fail token first - minimalVersion: "v0.2.0", - mockTokenSuccess: false, - expectedStatus: http.StatusUnauthorized, // 401 - expectCookie: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // --- Setup Mocks and Config for this test case --- - token := "mock-token" - - if tt.mockTokenSuccess { - auth.GetToken = func(_ http.Header) (string, int, error) { return token, http.StatusOK, nil } - auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { return &auth.Visas{}, nil } - auth.GetPermissions = func(_ auth.Visas) []string { return []string{"dataset1"} } - session.NewSessionKey = func() string { return "key" } - config.Config.Session.Name = "sda_session_key" - } else { - auth.GetToken = func(_ http.Header) (string, int, error) { - return "", http.StatusUnauthorized, errors.New("missing token") - } - auth.GetVisas = func(_ auth.OIDCDetails, _ string) (*auth.Visas, error) { - return &auth.Visas{}, errors.New("auth failed") - } - } - - // Set up ClientVersionMiddleware config - parsedVersion, err := semver.NewVersion(tt.minimalVersion) - assert.NoErrorf(t, err, "Test setup error: Failed to parse minimal version '%s'", tt.minimalVersion) - config.Config.App.MinimalCliVersion = parsedVersion - - // --- Execution --- - - w := httptest.NewRecorder() - r := httptest.NewRequest("GET", "/", nil) - if tt.clientVersion != "" { - r.Header.Set("SDA-Client-Version", tt.clientVersion) - } - - _, router := gin.CreateTestContext(w) - router.GET("/", append(ChainDefaultMiddleware(), testEndpoint)...) - router.ServeHTTP(w, r) - - // Assert Status Code - assert.Equal(t, tt.expectedStatus, w.Code, fmt.Sprintf("Expected status %d, got %d", tt.expectedStatus, w.Code)) - - // Assert Cookie Presence - cookieFound := false - for _, c := range w.Result().Cookies() { - if c.Name == config.Config.Session.Name { - cookieFound = true - - break - } - } - - if tt.expectCookie { - assert.True(t, cookieFound, "Expected a session cookie to be set, but none was found.") - } else { - assert.False(t, cookieFound, "Expected no session cookie to be set, but one was found.") - } - }) - } -} From 1927dc8e63fba49ec28fdb3ecdc7f0e642f7b4f6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 3 Nov 2025 16:10:07 +0100 Subject: [PATCH 149/184] feat: improve name and logic for middleware --- sda-download/api/api.go | 8 +++----- sda-download/dev_utils/config-notls.yaml | 1 + sda-download/dev_utils/config.yaml | 2 +- sda-download/internal/config/config.go | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 85bba3d9a..1c8f1816f 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -17,18 +17,16 @@ import ( ) // SelectedMiddleware returns the middleware chain based on configuration. -// For example, config.Config.App.Middleware could be "default", "token", etc. +// For example, config.Config.App.Middleware could be "default", "token-clientversion", etc. var SelectedMiddleware = func() []gin.HandlerFunc { switch strings.ToLower(config.Config.App.Middleware) { - case "default": + case "token-clientversion": return []gin.HandlerFunc{ middleware.TokenMiddleware(), middleware.ClientVersionMiddleware(), } - case "token": - return []gin.HandlerFunc{middleware.TokenMiddleware()} default: - return nil + return []gin.HandlerFunc{middleware.TokenMiddleware()} } } diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 4752b5488..63f8e2e89 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -1,4 +1,5 @@ app: + middleware: "token-clientversion" minimalcliversion: "v0.2.0" log: diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index 91bb62cc5..4dea553fb 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -3,7 +3,7 @@ app: servercert: "./dev_utils/certs/download.pem" serverkey: "./dev_utils/certs/download-key.pem" port: "8443" - middleware: "default" + middleware: "token-clientversion" minimalcliversion: "v0.2.0" log: diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index cba7baeca..18517aec6 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -25,7 +25,7 @@ const S3 = "s3" // availableMiddlewares list the options for middlewares // empty string "" is an alias for default, for when the config key is not set, or it's empty -var availableMiddlewares = []string{"", "default", "token"} +var availableMiddlewares = []string{"", "default", "token-clientversion"} // Config is a global configuration value store var Config Map From 754a908f2ab799393402b820811d745cc43c1c09 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 4 Nov 2025 11:48:45 +0100 Subject: [PATCH 150/184] feat: log error message to debug --- sda-download/api/middleware/middleware.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index 6f5e4fb77..94f6106c4 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -96,7 +96,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { // Check if the header is present if clientVersionStr == "" { - log.Warnf("request blocked (412): Missing client version header in request") + log.Debugln("request blocked (412): Missing client version header in request") c.String(http.StatusPreconditionFailed, "Missing client version header in request") c.Abort() @@ -106,7 +106,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { // Parse the client's provided version (using the processed string) clientVersion, err := semver.NewVersion(clientVersionStr) if err != nil { - log.Warnf("client version header '%s' is not a valid semantic version: %v", clientVersionStr, err) + log.Debugf("client version header '%s' is not a valid semantic version: %v", clientVersionStr, err) c.String(http.StatusPreconditionFailed, "client version header is not a valid semantic version") c.Abort() @@ -116,7 +116,7 @@ func ClientVersionMiddleware() gin.HandlerFunc { // Check if the client version is sufficient (clientVersion >= minimalVersion) if clientVersion.LessThan(config.Config.App.MinimalCliVersion) { errorMessage := fmt.Sprintf("Error: Your sda-cli client version is outdated, please update to at least version '%s'.", config.Config.App.MinimalCliVersionStr) - log.Warnf("request blocked (412): outdated client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) + log.Debugf("request blocked (412): outdated client version '%s'. Required minimum '%s'", clientVersionStr, config.Config.App.MinimalCliVersionStr) c.String(http.StatusPreconditionFailed, errorMessage) c.Abort() From 8d45b773a8a97212a4dc8fcf824565d7f46b6983 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:47:41 +0000 Subject: [PATCH 151/184] Bump github.com/opencontainers/runc Bumps the go_modules group with 1 update in the /sda directory: [github.com/opencontainers/runc](https://github.com/opencontainers/runc). Updates `github.com/opencontainers/runc` from 1.2.3 to 1.2.8 - [Release notes](https://github.com/opencontainers/runc/releases) - [Changelog](https://github.com/opencontainers/runc/blob/v1.2.8/CHANGELOG.md) - [Commits](https://github.com/opencontainers/runc/compare/v1.2.3...v1.2.8) --- updated-dependencies: - dependency-name: github.com/opencontainers/runc dependency-version: 1.2.8 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 4 ++-- sda/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 53d8a82e0..dc81e5ad7 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -129,13 +129,13 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.2.3 // indirect + github.com/opencontainers/runc v1.2.8 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.1 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect diff --git a/sda/go.sum b/sda/go.sum index ffac05408..848fbad75 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -284,8 +284,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= -github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= +github.com/opencontainers/runc v1.2.8 h1:RnEICeDReapbZ5lZEgHvj7E9Q3Eex9toYmaGBsbvU5Q= +github.com/opencontainers/runc v1.2.8/go.mod h1:cC0YkmZcuvr+rtBZ6T7NBoVbMGNAdLa/21vIElJDOzI= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= @@ -307,8 +307,8 @@ github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2e github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= From 8e649208cb35a03e28cee951502667d5ea34774b Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 7 Nov 2025 08:39:01 +0000 Subject: [PATCH 152/184] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 2b3d50db3..61a2ff228 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.21 -appVersion: v3.0.26 +version: 2.0.22 +appVersion: v3.0.31 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index a8abe33d8..2ca982151 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.21 -appVersion: v3.0.26 +version: 2.0.22 +appVersion: v3.0.31 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 6a1ec3ddb..4ff16bb6e 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.14 -appVersion: v3.0.26 +version: 3.0.15 +appVersion: v3.0.31 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 9dd6a7b1072a5060167571abe68df3d892363635 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:04:53 +0000 Subject: [PATCH 153/184] Bump golangci/golangci-lint-action from 8.0.0 to 9.0.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-version: 9.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 03d87c40b..c018b4a61 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -46,7 +46,7 @@ jobs: uses: actions/checkout@v5 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8.0.0 + uses: golangci/golangci-lint-action@v9.0.0 with: args: --timeout 5m working-directory: sda-download @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v5 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8.0.0 + uses: golangci/golangci-lint-action@v9.0.0 with: args: --timeout 5m working-directory: sda @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@v5 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v8.0.0 + uses: golangci/golangci-lint-action@v9.0.0 with: args: --timeout 5m working-directory: sda-admin From d91b448c36c29f06f22a49c4dd3f1f4c1ce3a6c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:05:37 +0000 Subject: [PATCH 154/184] Bump commons-io:commons-io in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [commons-io:commons-io](https://github.com/apache/commons-io). Updates `commons-io:commons-io` from 2.20.0 to 2.21.0 - [Changelog](https://github.com/apache/commons-io/blob/master/RELEASE-NOTES.txt) - [Commits](https://github.com/apache/commons-io/compare/rel/commons-io-2.20.0...rel/commons-io-2.21.0) --- updated-dependencies: - dependency-name: commons-io:commons-io dependency-version: 2.21.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 423f5c3e0..114218151 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -103,7 +103,7 @@ commons-io commons-io - 2.20.0 + 2.21.0 com.amazonaws From 058c9f03f4b308ca2ffe74e41fd421be8845b24f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Nov 2025 19:12:35 +0000 Subject: [PATCH 155/184] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.36 to 3.0.38 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.36...crypt4gh-3.0.38) Updates `no.elixir:clearinghouse` from 3.0.10 to 3.0.12 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.10...crypt4gh-3.0.12) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.38 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index e5005426d..a90f8f79a 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.36 + 3.0.38 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.10 + 3.0.12 org.slf4j From 56a8d5f902c77d533fa35b6460726d89e268fa34 Mon Sep 17 00:00:00 2001 From: Kostas Koumpouras <47719735+kostas-kou@users.noreply.github.com> Date: Tue, 11 Nov 2025 17:11:04 +0100 Subject: [PATCH 156/184] Update sda-admin/helpers/helpers.go Co-authored-by: Joakim Bygdell --- sda-admin/helpers/helpers.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index 0fc70af29..ed5d108aa 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -59,15 +59,7 @@ var PostRequest = PostReq // PostReq sends a POST request to the server with a JSON body and returns the response body or an error. func PostReq(url, token string, jsonBody []byte) ([]byte, error) { - var req *http.Request - var err error - if jsonBody != nil { - // Create a new POST request with the provided JSON body - req, err = http.NewRequest("POST", url, bytes.NewBuffer(jsonBody)) - } else { - // Create a new POST request with query - req, err = http.NewRequest("POST", url, nil) - } + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBody)) if err != nil { return nil, fmt.Errorf("failed to create the request, reason: %v", err) } From a600687fcc459a46476c4d4bceecc69a1cecad47 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 12 Nov 2025 12:59:36 +0000 Subject: [PATCH 157/184] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 61a2ff228..360dc6f37 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.22 -appVersion: v3.0.31 +version: 2.0.23 +appVersion: v3.0.33 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 2ca982151..29e2fd39c 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.22 -appVersion: v3.0.31 +version: 2.0.23 +appVersion: v3.0.33 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 4ff16bb6e..a409456e4 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.15 -appVersion: v3.0.31 +version: 3.0.16 +appVersion: v3.0.33 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 28d1c610ee844c8e58a262660e495dada8e96293 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:05:32 +0000 Subject: [PATCH 158/184] build(deps): bump golang.org/x/crypto Bumps the all-modules group in /sda-download with 1 update: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.43.0 to 0.44.0 - [Commits](https://github.com/golang/crypto/compare/v0.43.0...v0.44.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.44.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 14 +++++++------- sda-download/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c94c74b6c..6781c7213 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -16,7 +16,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.43.0 + golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.10 @@ -72,12 +72,12 @@ require ( go.uber.org/mock v0.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/mod v0.28.0 // indirect - golang.org/x/net v0.45.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.30.0 // indirect - golang.org/x/tools v0.37.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.46.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 8ad3a676e..d1b3fa63a 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -181,28 +181,28 @@ golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= +golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= -golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -214,31 +214,31 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= -golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= From 29fb12263d22403de56c4b6d1822fa6d2464a2c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Nov 2025 19:14:10 +0000 Subject: [PATCH 159/184] build(deps): bump com.squareup.okhttp3:okhttp-jvm Bumps the all-modules group in /sda-doa with 1 update: [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp). Updates `com.squareup.okhttp3:okhttp-jvm` from 5.3.0 to 5.3.1 - [Changelog](https://github.com/square/okhttp/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okhttp/compare/parent-5.3.0...parent-5.3.1) --- updated-dependencies: - dependency-name: com.squareup.okhttp3:okhttp-jvm dependency-version: 5.3.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index a90f8f79a..01288956c 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -137,7 +137,7 @@ com.squareup.okhttp3 okhttp-jvm - 5.3.0 + 5.3.1 From 9999cfc55e5bcbee565ab7f9a71563b921d7b9ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 01:37:35 +0000 Subject: [PATCH 160/184] build(deps): bump golang.org/x/crypto Bumps the go_modules group with 1 update in the /sda-admin directory: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.35.0 to 0.45.0 - [Commits](https://github.com/golang/crypto/compare/v0.35.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-admin/go.mod | 4 ++-- sda-admin/go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sda-admin/go.mod b/sda-admin/go.mod index bb272c942..6a388feef 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -14,7 +14,7 @@ require ( github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/crypto v0.35.0 // indirect - golang.org/x/sys v0.30.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/sys v0.38.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-admin/go.sum b/sda-admin/go.sum index b244f10f2..b9308fa25 100644 --- a/sda-admin/go.sum +++ b/sda-admin/go.sum @@ -14,12 +14,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From eed51dd9adc1488011dde17f4dadcb5c7de1c1bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 02:48:41 +0000 Subject: [PATCH 161/184] build(deps): bump golang.org/x/crypto from 0.42.0 to 0.45.0 in /sda Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.42.0 to 0.45.0. - [Commits](https://github.com/golang/crypto/compare/v0.42.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sda/go.mod | 14 +++++++------- sda/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index dc81e5ad7..6ef93e729 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -31,7 +31,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.42.0 + golang.org/x/crypto v0.45.0 golang.org/x/oauth2 v0.31.0 google.golang.org/grpc v1.75.1 google.golang.org/protobuf v1.36.9 @@ -164,13 +164,13 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/mod v0.27.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.36.0 // indirect - golang.org/x/text v0.29.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.11.0 // indirect - golang.org/x/tools v0.36.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 848fbad75..25da9c612 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -433,8 +433,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -445,8 +445,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -462,8 +462,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -476,8 +476,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -501,8 +501,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -512,8 +512,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -524,8 +524,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -539,8 +539,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From a389c9e7e659667d919f40e9e749f787432332a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Nov 2025 03:23:34 +0000 Subject: [PATCH 162/184] build(deps): bump golang.org/x/crypto in /sda-download Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.44.0 to 0.45.0. - [Commits](https://github.com/golang/crypto/compare/v0.44.0...v0.45.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.45.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 ++-- sda-download/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 6781c7213..c865263ba 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -16,7 +16,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.44.0 + golang.org/x/crypto v0.45.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.10 @@ -73,7 +73,7 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/mod v0.29.0 // indirect - golang.org/x/net v0.46.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index d1b3fa63a..b6820e41a 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -181,8 +181,8 @@ golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -196,8 +196,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 7a750610d9a5a594be8d89bc2a57c2618a1b663d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 19:37:41 +0000 Subject: [PATCH 163/184] build(deps): bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 18 +++++++++--------- .github/workflows/code-linter.yaml | 8 ++++---- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/functionality.yml | 6 +++--- .github/workflows/publish_charts.yml | 2 +- .github/workflows/publish_container.yml | 6 +++--- .github/workflows/release_sda-admin.yaml | 4 ++-- .github/workflows/shellcheck.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index be22308a9..889c99dab 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -25,7 +25,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -69,7 +69,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -149,7 +149,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -223,7 +223,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Test rabbitmq federation run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @@ -235,7 +235,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Test postgres run: docker compose -f .github/integration/postgres.yml run tests @@ -250,7 +250,7 @@ jobs: storage: ["posix", "s3"] steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Test sensitive-data-archive run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test @@ -261,7 +261,7 @@ jobs: - build_server_images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: @@ -284,7 +284,7 @@ jobs: storage: [s3, posix] steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Test sda-doa for ${{ matrix.storage }} storage run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test @@ -310,7 +310,7 @@ jobs: storage: "posix" steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install Helm uses: azure/setup-helm@v4 diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index c018b4a61..459bf78eb 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -18,7 +18,7 @@ jobs: sda: ${{ steps.changes.outputs.sda }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: @@ -43,7 +43,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Run golangci-lint uses: golangci/golangci-lint-action@v9.0.0 @@ -64,7 +64,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Run golangci-lint uses: golangci/golangci-lint-action@v9.0.0 @@ -85,7 +85,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Run golangci-lint uses: golangci/golangci-lint-action@v9.0.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6080a5951..059043c67 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index 5e6cbca4f..739d7af57 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -11,7 +11,7 @@ jobs: sftp-inbox: ${{ steps.changes.outputs.sftp-inbox }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: dorny/paths-filter@v3 id: changes with: @@ -40,7 +40,7 @@ jobs: python-version: "3.11" - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Run setup scripts run: | @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Build image run: | diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index 06c0dffb3..7c8fdf96d 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -16,7 +16,7 @@ jobs: continue-on-error: true steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index cd0e7f957..f44a7d5ab 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -24,7 +24,7 @@ jobs: new_tag: ${{ steps.bump_tag.outputs.new_tag }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: '0' - name: Bump version and push tag @@ -46,7 +46,7 @@ jobs: packages: write steps: - name: Check out the repo - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -101,7 +101,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: '0' diff --git a/.github/workflows/release_sda-admin.yaml b/.github/workflows/release_sda-admin.yaml index 43c7f1d38..5df7953ab 100644 --- a/.github/workflows/release_sda-admin.yaml +++ b/.github/workflows/release_sda-admin.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Get version tag run: | VERSION=$(cat sda-admin/.version) @@ -37,7 +37,7 @@ jobs: - "darwin/arm64" steps: - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Go uses: actions/setup-go@v6 diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index caf7e1e37..fba3ef920 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: check all scripts uses: ludeeus/action-shellcheck@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f666ba6e..2a676fbde 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v5 with: @@ -44,7 +44,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Get dependencies run: | @@ -79,7 +79,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Get dependencies run: | @@ -114,7 +114,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Get dependencies run: | From 2ce25bf000737d57cbce66acb969731b68f3f6fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Nov 2025 07:47:18 +0000 Subject: [PATCH 164/184] build(deps): bump golangci/golangci-lint-action from 9.0.0 to 9.1.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 9.0.0 to 9.1.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v9.0.0...v9.1.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-version: 9.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 459bf78eb..93bf2ffa2 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -46,7 +46,7 @@ jobs: uses: actions/checkout@v6 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v9.0.0 + uses: golangci/golangci-lint-action@v9.1.0 with: args: --timeout 5m working-directory: sda-download @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v6 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v9.0.0 + uses: golangci/golangci-lint-action@v9.1.0 with: args: --timeout 5m working-directory: sda @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@v6 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v9.0.0 + uses: golangci/golangci-lint-action@v9.1.0 with: args: --timeout 5m working-directory: sda-admin From ab06e9dab0856d09e88bef3b2ae0f38cd381af5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Nov 2025 14:15:38 +0000 Subject: [PATCH 165/184] build(deps): bump google.golang.org/grpc Bumps the all-modules group with 1 update in the /sda-download directory: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.76.0 to 1.77.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.76.0...v1.77.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.77.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 ++-- sda-download/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c865263ba..1ce8f5894 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -18,7 +18,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.45.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.76.0 + google.golang.org/grpc v1.77.0 google.golang.org/protobuf v1.36.10 ) @@ -78,7 +78,7 @@ require ( golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.31.0 // indirect golang.org/x/tools v0.38.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index b6820e41a..ab80b1378 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -161,18 +161,18 @@ github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -242,10 +242,10 @@ golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 6734a7cbec6b70519d13c70a81ed2003bb009ba5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 19:56:20 +0000 Subject: [PATCH 166/184] build(deps): bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [com.squareup.okhttp3:okhttp-jvm](https://github.com/square/okhttp). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.7 to 4.0.0 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.7...v4.0.0) Updates `com.squareup.okhttp3:okhttp-jvm` from 5.3.1 to 5.3.2 - [Changelog](https://github.com/square/okhttp/blob/master/CHANGELOG.md) - [Commits](https://github.com/square/okhttp/compare/parent-5.3.1...parent-5.3.2) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 4.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: all-modules - dependency-name: com.squareup.okhttp3:okhttp-jvm dependency-version: 5.3.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 01288956c..f9f44e60f 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.7 + 4.0.0 no.uio.ifi @@ -137,7 +137,7 @@ com.squareup.okhttp3 okhttp-jvm - 5.3.1 + 5.3.2 From 71f59ce321e5d1e16d1714a098a6b702178e36fa Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 27 Nov 2025 12:19:43 +0100 Subject: [PATCH 167/184] replace gson with spring-boot-starter-gson --- sda-doa/pom.xml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index f9f44e60f..131aab4e9 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -18,11 +18,6 @@ - - com.google.code.gson - gson - 2.13.2 - org.springframework.boot spring-boot-starter-data-jpa @@ -35,6 +30,10 @@ org.springframework.boot spring-boot-starter-amqp + + org.springframework.boot + spring-boot-starter-gson + org.postgresql From 9a78ecd39901f632dab2426a7a603e28d197158b Mon Sep 17 00:00:00 2001 From: neicnordic Date: Thu, 27 Nov 2025 15:29:21 +0000 Subject: [PATCH 168/184] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 360dc6f37..2159b26f4 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.23 -appVersion: v3.0.33 +version: 2.0.24 +appVersion: v3.0.39 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 29e2fd39c..2b085f65b 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.23 -appVersion: v3.0.33 +version: 2.0.24 +appVersion: v3.0.39 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index a409456e4..4baab1511 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.16 -appVersion: v3.0.33 +version: 3.0.17 +appVersion: v3.0.39 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 9eb80cb1d2b8e8d8c423c2f70a18db8da9d80b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 19 Nov 2025 16:00:15 +0100 Subject: [PATCH 169/184] feat(postgresql): update init of db to add new indexes and colum changes on files and file_event_log tables, add migration steps to reflect indexes and columns changes, and deprecation of file_event_log.correlation_id with data migration --- postgresql/initdb.d/01_main.sql | 7 +- postgresql/initdb.d/02_functions.sql | 34 ++++---- ...19_add_indexes_and_new_column_on_files.sql | 27 ++++++ ...precate_files_event_log_correlation_id.sql | 82 +++++++++++++++++++ 4 files changed, 132 insertions(+), 18 deletions(-) create mode 100644 postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql create mode 100644 postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index fc931b49e..c007e904c 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -58,6 +58,8 @@ CREATE TABLE files ( submission_user TEXT, submission_file_path TEXT DEFAULT '' NOT NULL, + submission_file_root_dir TEXT GENERATED ALWAYS AS (split_part(submission_file_path, '/', 1)) STORED, + submission_file_size BIGINT, archive_file_path TEXT DEFAULT '' NOT NULL, archive_file_size BIGINT, @@ -76,6 +78,8 @@ CREATE TABLE files ( CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user) ); +-- Add indexes to the files table +CREATE INDEX files_submission_user_submission_file_root_dir_idx ON sda.files(submission_user, submission_file_root_dir); -- The user info is used by auth to be able to link users to their name and email CREATE TABLE userinfo ( @@ -151,7 +155,6 @@ CREATE TABLE file_event_log ( id SERIAL PRIMARY KEY, file_id UUID REFERENCES files(id), event TEXT REFERENCES file_events(title), - correlation_id UUID, -- Correlation ID in the message's header user_id TEXT, -- Elixir user id (or pipeline-step for ingestion, -- etc.) details JSONB, -- This is my solution to fields such as @@ -165,6 +168,8 @@ CREATE TABLE file_event_log ( success BOOLEAN, error TEXT ); +-- Add indexes to the file_event_log table +CREATE INDEX file_event_log_file_id_started_at_idx ON sda.file_event_log(file_id, started_at); -- This table is used to define events for dataset event logging. CREATE TABLE dataset_events ( diff --git a/postgresql/initdb.d/02_functions.sql b/postgresql/initdb.d/02_functions.sql index 96355ba1a..b80231793 100644 --- a/postgresql/initdb.d/02_functions.sql +++ b/postgresql/initdb.d/02_functions.sql @@ -18,32 +18,32 @@ CREATE TRIGGER files_last_modified EXECUTE PROCEDURE files_updated(); -- Function for registering files on upload -CREATE FUNCTION register_file(submission_file_path TEXT, submission_user TEXT) -RETURNS TEXT AS $register_file$ +CREATE FUNCTION sda.register_file(file_id TEXT, submission_file_path TEXT, submission_user TEXT) + RETURNS TEXT AS $register_file$ DECLARE - file_ext TEXT; file_uuid UUID; BEGIN -- Upsert file information. we're not interested in restarted uploads so old -- overwritten files that haven't been ingested are updated instead of -- inserting a new row. - INSERT INTO sda.files( submission_file_path, submission_user, encryption_method ) - VALUES( submission_file_path, submission_user, 'CRYPT4GH' ) - ON CONFLICT ON CONSTRAINT unique_ingested - DO UPDATE SET submission_file_path = EXCLUDED.submission_file_path, - submission_user = EXCLUDED.submission_user, - encryption_method = EXCLUDED.encryption_method - RETURNING id INTO file_uuid; - - -- We add a new event for every registration though, as this might help for - -- debugging. - INSERT INTO sda.file_event_log( file_id, event, user_id ) - VALUES (file_uuid, 'registered', submission_user); - - RETURN file_uuid; +INSERT INTO sda.files( id, submission_file_path, submission_user, encryption_method ) +VALUES( COALESCE(CAST(NULLIF(file_id, '') AS UUID), gen_random_uuid()), submission_file_path, submission_user, 'CRYPT4GH' ) + ON CONFLICT ON CONSTRAINT unique_ingested + DO UPDATE SET submission_file_path = EXCLUDED.submission_file_path, + submission_user = EXCLUDED.submission_user, + encryption_method = EXCLUDED.encryption_method + RETURNING id INTO file_uuid; + +-- We add a new event for every registration though, as this might help for +-- debugging. +INSERT INTO sda.file_event_log( file_id, event, user_id ) +VALUES (file_uuid, 'registered', submission_user); + +RETURN file_uuid; END; $register_file$ LANGUAGE plpgsql; + CREATE FUNCTION set_archived(file_uuid UUID, corr_id UUID, file_path TEXT, file_size BIGINT, inbox_checksum_value TEXT, inbox_checksum_type TEXT) RETURNS void AS $set_archived$ BEGIN diff --git a/postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql b/postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql new file mode 100644 index 000000000..7825f2688 --- /dev/null +++ b/postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql @@ -0,0 +1,27 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 18; + changes VARCHAR := 'Create new indexes on files and file_event_log tables, and new generated column submission_file_root_dir on files table'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + + ALTER TABLE sda.files + ADD COLUMN submission_file_root_dir TEXT GENERATED ALWAYS AS (split_part(submission_file_path, '/', 1)) STORED; + + + CREATE INDEX files_submission_user_submission_file_root_dir_idx + ON sda.files(submission_user, submission_file_root_dir); + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ diff --git a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql new file mode 100644 index 000000000..bd90b2c29 --- /dev/null +++ b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql @@ -0,0 +1,82 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 17; + changes VARCHAR := 'Create rotatekey role and grant it priviledges to sda tables'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + -- Migrate data where files.id != file_event_log.correlation_id + + -- First drop foreign key constraint so we can update values without constraint restriction + ALTER TABLE sda.file_event_log + DROP CONSTRAINT file_event_log_file_id_fkey; + + -- Update all files which have a file_event_log where file_id != correlation_id + UPDATE sda.files AS f + SET id = fel.correlation_id + FROM sda.file_event_log AS fel + WHERE f.id = fel.file_id + AND fel.file_id != fel.correlation_id + AND fel.correlation_id IS NOT NULL; + + -- Update all file_event_log where file_id != correlation_id + UPDATE sda.file_event_log AS f + SET file_id = fel.correlation_id + FROM sda.file_event_log AS fel + WHERE f.file_id = fel.file_id + AND fel.file_id != fel.correlation_id + AND fel.correlation_id IS NOT NULL; + + -- Add back the foreign key constraint + ALTER TABLE sda.file_event_log + ADD CONSTRAINT file_event_log_file_id_fkey FOREIGN KEY (file_id) + REFERENCES sda.files(id); + + + -- Update RegisterFile func + -- First drop it so we can create the updated version + DROP FUNCTION IF EXISTS sda.register_file; + + + -- Create updated function + -- Function for registering files on upload + CREATE FUNCTION sda.register_file(file_id TEXT, submission_file_path TEXT, submission_user TEXT) + RETURNS TEXT AS $register_file$ + DECLARE + file_uuid UUID; + BEGIN + -- Upsert file information. we're not interested in restarted uploads so old + -- overwritten files that haven't been ingested are updated instead of + -- inserting a new row. + INSERT INTO sda.files( id, submission_file_path, submission_user, encryption_method ) + VALUES( COALESCE(CAST(NULLIF(file_id, '') AS UUID), gen_random_uuid()), submission_file_path, submission_user, 'CRYPT4GH' ) + ON CONFLICT ON CONSTRAINT unique_ingested + DO UPDATE SET submission_file_path = EXCLUDED.submission_file_path, + submission_user = EXCLUDED.submission_user, + encryption_method = EXCLUDED.encryption_method + RETURNING id INTO file_uuid; + + -- We add a new event for every registration though, as this might help for + -- debugging. + INSERT INTO sda.file_event_log( file_id, event, user_id ) + VALUES (file_uuid, 'registered', submission_user); + + RETURN file_uuid; + END; + $register_file$ LANGUAGE plpgsql; + + ALTER TABLE sda.file_event_log + DROP COLUMN correlation_id; + +ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ From a1006e320074cfb0a3374383357079d709740996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 19 Nov 2025 17:38:44 +0100 Subject: [PATCH 170/184] feat(sda-stack): WIP update sda stack to work with database changes --- .github/integration/sda-s3-integration.yml | 2 +- sda/cmd/api/api.go | 44 ++----- sda/cmd/finalize/finalize.go | 23 +--- sda/cmd/ingest/ingest.go | 64 ++++------ sda/cmd/s3inbox/proxy.go | 49 +++++--- sda/cmd/verify/verify.go | 8 +- sda/internal/database/database.go | 5 +- sda/internal/database/db_functions.go | 135 ++++++--------------- 8 files changed, 114 insertions(+), 216 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 3567b9600..e18b91666 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -11,7 +11,7 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: python:3.11-slim + image: python:3.11-slim-bookworm volumes: - ./scripts:/scripts - shared:/shared diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 0b2d76ba0..9501e9b24 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -328,7 +328,7 @@ and sends it to the broker with the appropriate correlation ID. func ingestFile(c *gin.Context) { var ( ingest schema.IngestionTrigger - corrID string + fileID string ) switch { case c.Query("fileid") != "" && c.Request.ContentLength > 0: @@ -346,7 +346,7 @@ func ingestFile(c *gin.Context) { // Add file info in the message payload ingest.User = fileDetails.User ingest.FilePath = fileDetails.Path - corrID = fileDetails.CorrID + case c.Request.ContentLength > 0: // Bind ingest and payload if err = c.BindJSON(&ingest); err != nil { @@ -361,9 +361,9 @@ func ingestFile(c *gin.Context) { return } // Find the correlation id of the file - corrID, err = Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") + fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(ingest.User, ingest.FilePath, "uploaded") if err != nil { - if corrID == "" { + if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -386,7 +386,7 @@ func ingestFile(c *gin.Context) { return } - err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "ingest", marshaledMsg) + err = Conf.API.MQ.SendMessage(fileID, Conf.Broker.Exchange, "ingest", marshaledMsg) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -443,7 +443,7 @@ func deleteFile(c *gin.Context) { time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } - if err := Conf.API.DB.UpdateFileEventLog(fileID, "disabled", fileID, "api", "{}", "{}"); err != nil { + if err := Conf.API.DB.UpdateFileEventLog(fileID, "disabled", "api", "{}", "{}"); err != nil { log.Errorf("set status deleted failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -581,7 +581,7 @@ The function constructs an accession message, validates it and sends it to the m func setAccession(c *gin.Context) { var ( accession schema.IngestionAccession - corrID string + fileID string ) hasQuery := c.Query("fileid") != "" || c.Query("accessionid") != "" missingAccession := c.Query("fileid") != "" && c.Query("accessionid") == "" @@ -616,8 +616,7 @@ func setAccession(c *gin.Context) { accession.User = fileDetails.User accession.FilePath = fileDetails.Path accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} - // Corellation id - corrID = fileDetails.CorrID + case c.Request.ContentLength > 0: if err = c.BindJSON(&accession); err != nil { c.AbortWithStatusJSON( @@ -631,7 +630,7 @@ func setAccession(c *gin.Context) { return } // Find the correlation id - fileID, err := Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "verified") + fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "verified") if err != nil { if fileID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) @@ -641,17 +640,6 @@ func setAccession(c *gin.Context) { return } - // Get correlation id - corrID, err = Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") - if err != nil { - if corrID == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - } else { - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - } - - return - } // Get decrypted checksum fileDecrChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileID) if err != nil { @@ -678,7 +666,7 @@ func setAccession(c *gin.Context) { return } - err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "accession", marshaledMsg) + err = Conf.API.MQ.SendMessage(fileID, Conf.Broker.Exchange, "accession", marshaledMsg) if err != nil { log.Debugln(err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -725,8 +713,7 @@ func createDataset(c *gin.Context) { return } } - _, err = Conf.API.DB.GetCorrID(dataset.User, inboxPath, stableID) - if err != nil { + if _, err := Conf.API.DB.GetFileIDByUserPathAndStatus(dataset.User, inboxPath, "ready"); err != nil { switch { case err.Error() == "sql: no rows in result set": log.Errorln(err.Error()) @@ -1008,13 +995,6 @@ func reVerify(c *gin.Context, accessionID string) (*gin.Context, error) { return c, err } - corrID, err := Conf.API.DB.GetCorrID(reVerify.User, reVerify.FilePath, accessionID) - if err != nil { - log.Errorf("failed to get CorrID for %s, %s", reVerify.User, reVerify.FilePath) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return c, err - } marshaledMsg, _ := json.Marshal(&reVerify) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { @@ -1024,7 +1004,7 @@ func reVerify(c *gin.Context, accessionID string) (*gin.Context, error) { return c, err } - err = Conf.API.MQ.SendMessage(corrID, Conf.Broker.Exchange, "archived", marshaledMsg) + err = Conf.API.MQ.SendMessage(reVerify.FileID, Conf.Broker.Exchange, "archived", marshaledMsg) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 8f7ca8acf..486c9162a 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -106,8 +106,7 @@ func main() { continue - case "verified": - case "enabled": + case "verified", "enabled": case "ready": log.Infof("File with correlation-id: %s is already marked as ready.", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { @@ -124,16 +123,7 @@ func main() { continue } - fileID, err := db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get file-id for file with correlation-id: %s, reason: %v", delivered.CorrelationId, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - - continue - } - + fileID := delivered.CorrelationId c := schema.IngestionCompletion{ User: message.User, FilePath: message.FilePath, @@ -204,7 +194,7 @@ func main() { } // Mark file as "ready" - if err := db.UpdateFileEventLog(fileID, "ready", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(fileID, "ready", "finalize", "{}", string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) @@ -233,10 +223,7 @@ func main() { func backupFile(delivered amqp.Delivery) error { log.Debug("Backup initiated") - fileID, err := db.GetFileID(delivered.CorrelationId) - if err != nil { - return fmt.Errorf("failed to get ID for file, reason: %s", err.Error()) - } + fileID := delivered.CorrelationId filePath, fileSize, err := db.GetArchived(fileID) if err != nil { @@ -272,7 +259,7 @@ func backupFile(delivered amqp.Delivery) error { } // Mark file as "backed up" - if err := db.UpdateFileEventLog(fileID, "backed up", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(fileID, "backed up", "finalize", "{}", string(delivered.Body)); err != nil { return fmt.Errorf("UpdateFileEventLog failed, reason: (%v)", err) } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 5c6492356..772dfceef 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -201,19 +201,10 @@ func (app *Ingest) registerC4GHKey() error { return nil } -func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { - fileID, err := app.DB.GetFileID(correlationID) - if err != nil { - log.Errorf("failed to get file-id for file from message (correlation-id: %s), reason: %s", correlationID, err.Error()) - if strings.Contains(err.Error(), "sql: no rows in result set") { - return "reject" - } - - return "nack" - } +func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) string { m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { + if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to update event log for file with id : %s", fileID) return "nack" @@ -222,23 +213,17 @@ func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrig return "ack" } -func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrigger) string { - var fileID string - status, err := app.DB.GetFileStatus(correlationID) +func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) string { + + status, err := app.DB.GetFileStatus(fileID) if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) + log.Errorf("failed to get status for file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" } switch status { case "disabled": - fileID, err = app.DB.GetFileID(correlationID) - if err != nil { - log.Errorf("failed to get file-id for file, correlation-id: %s, reason: %s", correlationID, err.Error()) - - return "nack" - } fileInfo, err := app.DB.GetFileInfo(fileID) if err != nil { @@ -257,7 +242,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig log.Errorf("Failed to open file to ingest, file-id: %s, inbox path: %s, reason: (%s)", fileID, message.FilePath, err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. @@ -267,7 +252,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -307,13 +292,13 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } m, _ := json.Marshal(message) - if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", correlationID, "ingest", "{}", string(m)); err != nil { + if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to set ingestion status for file from message, file-id: %s", fileID) return "nack" } - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -323,22 +308,17 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } case "": // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox - log.Infof("registering file, correlation-id: %s", correlationID) - fileID, err = app.DB.RegisterFile(message.FilePath, message.User) + log.Infof("registering file, correlation-id: %s", fileID) + fileID, err = app.DB.RegisterFile(&fileID, message.FilePath, message.User) if err != nil { - log.Errorf("failed to register file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) + log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" } case "uploaded": - fileID, err = app.DB.GetFileID(correlationID) - if err != nil { - log.Errorf("failed to get ID for file, correlation-id: %s, reason: %s", correlationID, err.Error()) - return "nack" - } default: - log.Warnf("unsupported file status: %s, correlation-id: %s", status, correlationID) + log.Warnf("unsupported file status: %s, correlation-id: %s", status, fileID) return "reject" } @@ -350,7 +330,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. @@ -360,7 +340,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -389,7 +369,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } m, _ := json.Marshal(message) - if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { + if err = app.DB.UpdateFileEventLog(fileID, "submitted", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } @@ -444,7 +424,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig if privateKey == nil { log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } @@ -455,7 +435,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) } @@ -535,7 +515,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig log.Debugf("Wrote archived file (file-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", fileID, message.User, message.FilePath, fileID, fileInfo.Size) - status, err = app.DB.GetFileStatus(correlationID) + status, err = app.DB.GetFileStatus(fileID) if err != nil { log.Errorf("failed to get file status, file-id: %s, reason: (%s)", fileID, err.Error()) @@ -554,7 +534,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "nack" } - if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { + if err := app.DB.UpdateFileEventLog(fileID, "archived", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to set event log status for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" @@ -580,7 +560,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "nack" } - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism log.Errorf("failed to publish message, reason: %v", err) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 0604ec688..ee0d956e0 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -29,6 +29,10 @@ import ( log "github.com/sirupsen/logrus" ) +type uniqueFileID struct { + username, filePath string +} + // Proxy represents the toplevel object in this application type Proxy struct { s3 storage.S3Conf @@ -36,7 +40,7 @@ type Proxy struct { messenger *broker.AMQPBroker database *database.SDAdb client *http.Client - fileIDs map[string]string + fileIDs map[uniqueFileID]string } // The Event struct @@ -82,7 +86,7 @@ func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *bro tr := &http.Transport{TLSClientConfig: tlsConf} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} - return &Proxy{s3conf, auth, messenger, db, client, make(map[string]string)} + return &Proxy{s3conf, auth, messenger, db, client, make(map[uniqueFileID]string)} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -160,12 +164,19 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } + fileIdentifier := uniqueFileID{ + username: username, + filePath: filepath, + } + + log.Infof("fileidentifier: %v, fileId: %s", fileIdentifier, p.fileIDs[fileIdentifier]) + // if this is an upload request - if p.detectRequestType(r) == Put && p.fileIDs[r.URL.Path] == "" { + if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { // register file in database log.Debugf("registering file %v in the database", r.URL.Path) - p.fileIDs[r.URL.Path], err = p.database.RegisterFile(filepath, username) - log.Debugf("fileId: %v", p.fileIDs[r.URL.Path]) + p.fileIDs[fileIdentifier], err = p.database.RegisterFile(nil, filepath, username) + log.Debugf("fileId: %v", p.fileIDs[fileIdentifier]) if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) @@ -174,7 +185,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // check if the file already exists, in that case send an overwrite message, // so that the FEGA portal is informed that a new version of the file exists. - err = p.sendMessageOnOverwrite(r, rawFilepath, token) + err = p.sendMessageOnOverwrite(p.fileIDs[fileIdentifier], r, rawFilepath, token) if err != nil { p.internalServerError(w, r, err.Error()) @@ -207,7 +218,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } - err = p.checkAndSendMessage(jsonMessage, r) + err = p.checkAndSendMessage(p.fileIDs[fileIdentifier], jsonMessage, r) if err != nil { p.internalServerError(w, r, fmt.Sprintf("broker error: %v", err)) @@ -216,33 +227,31 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // The following block is for treating the case when the client loses connection to the server and then it reconnects to a // different instance of s3inbox. For more details see #1358. - if p.fileIDs[r.URL.Path] == "" { - p.fileIDs[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") + if p.fileIDs[fileIdentifier] == "" { + p.fileIDs[fileIdentifier], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) return } - log.Debugf("resuming work on file with fileId: %v", p.fileIDs[r.URL.Path]) + log.Debugf("resuming work on file with fileId: %v", p.fileIDs[fileIdentifier]) } - if err := p.storeObjectSizeInDB(rawFilepath, p.fileIDs[r.URL.Path]); err != nil { + if err := p.storeObjectSizeInDB(rawFilepath, p.fileIDs[fileIdentifier]); err != nil { log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) p.internalServerError(w, r, "storeObjectSizeInDB failed") return } - log.Debugf("marking file %v as 'uploaded' in database", p.fileIDs[r.URL.Path]) - err = p.database.UpdateFileEventLog(p.fileIDs[r.URL.Path], "uploaded", p.fileIDs[r.URL.Path], "inbox", "{}", string(jsonMessage)) + log.Debugf("marking file %v as 'uploaded' in database", p.fileIDs[fileIdentifier]) + err = p.database.UpdateFileEventLog(p.fileIDs[fileIdentifier], "uploaded", "inbox", "{}", string(jsonMessage)) if err != nil { p.internalServerError(w, r, fmt.Sprintf("could not connect to db: %v", err)) return } - - delete(p.fileIDs, r.URL.Path) } // Writing non-200 to the response before the headers propagate the error @@ -276,7 +285,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } // Renew the connection to MQ if necessary, then send message -func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { +func (p *Proxy) checkAndSendMessage(fileID string, jsonMessage []byte, r *http.Request) error { var err error if p.messenger == nil { return errors.New("messenger is down") @@ -297,8 +306,8 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { } } - log.Debugf("Sending message with id %s", p.fileIDs[r.URL.Path]) - if err := p.messenger.SendMessage(p.fileIDs[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { + log.Debugf("Sending message with id %s", fileID) + if err := p.messenger.SendMessage(fileID, p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } @@ -562,7 +571,7 @@ func (p *Proxy) checkFileExists(fullPath string) (bool, error) { return result != nil, err } -func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, token jwt.Token) error { +func (p *Proxy) sendMessageOnOverwrite(fileID string, r *http.Request, rawFilepath string, token jwt.Token) error { exist, err := p.checkFileExists(r.URL.Path) if err != nil { return err @@ -583,7 +592,7 @@ func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, toke return err } - err = p.checkAndSendMessage(jsonMessage, r) + err = p.checkAndSendMessage(fileID, jsonMessage, r) if err != nil { return err } diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 572ac6c5c..c26d22bf4 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -161,7 +161,7 @@ func main() { log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", string(jsonMsg), string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } } @@ -273,7 +273,7 @@ func main() { if file.DecryptedChecksum != decrypted { log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) - if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%v)", err) @@ -290,7 +290,7 @@ func main() { if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) - if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%v)", err) @@ -377,7 +377,7 @@ func main() { log.Infof("file is already verified, file-id: %s", message.FileID) } - if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + if err := db.UpdateFileEventLog(message.FileID, "verified", "ingest", "{}", string(verifiedMessage)); err != nil { log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 82e7da664..71f6073b7 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -62,9 +62,8 @@ type DatasetInfo struct { } type FileDetails struct { - User string - Path string - CorrID string + User string + Path string } // SchemaName is the name of the remote database schema to query diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 72ad09eaf..3a9085b47 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -19,48 +19,29 @@ import ( // RegisterFile inserts a file in the database, along with a "registered" log // event. If the file already exists in the database, the entry is updated, but // a new file event is always inserted. -func (dbs *SDAdb) RegisterFile(uploadPath, uploadUser string) (string, error) { +// If fileId is provided the new files table row will have that id, otherwise a new uuid will be generated +// If the unique unique_ingested constraint(submission_file_path, archive_file_path, submission_user) already exists +// and a different fileId is provided, the fileId in the database will NOT be updated. +func (dbs *SDAdb) RegisterFile(fileID *string, uploadPath, uploadUser string) (string, error) { dbs.checkAndReconnectIfNeeded() if dbs.Version < 4 { return "", errors.New("database schema v4 required for RegisterFile()") } - query := "SELECT sda.register_file($1, $2);" + query := "SELECT sda.register_file($1, $2, $3);" - var fileID string - - err := dbs.DB.QueryRow(query, uploadPath, uploadUser).Scan(&fileID) - - return fileID, err -} - -func (dbs *SDAdb) GetFileID(corrID string) (string, error) { - var ( - err error - count int - ID string - ) + var createdFileId string - for count == 0 || (err != nil && count < RetryTimes) { - ID, err = dbs.getFileID(corrID) - count++ + fileIDArg := sql.NullString{} + if fileID != nil { + fileIDArg.Valid = true + fileIDArg.String = *fileID } - return ID, err -} -func (dbs *SDAdb) getFileID(corrID string) (string, error) { - dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const getFileID = "SELECT DISTINCT file_id FROM sda.file_event_log where correlation_id = $1;" + err := dbs.DB.QueryRow(query, fileIDArg, uploadPath, uploadUser).Scan(&createdFileId) - var fileID string - err := db.QueryRow(getFileID, corrID).Scan(&fileID) - if err != nil { - return "", err - } - - return fileID, nil + return createdFileId, err } // GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID @@ -140,26 +121,29 @@ AS subquery WHERE event = $3);` // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. -func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, corrID, user, details, message string) error { +func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, user, details, message string) error { var ( err error count int ) for count == 0 || (err != nil && count < RetryTimes) { - err = dbs.updateFileEventLog(fileUUID, event, corrID, user, details, message) + err = dbs.updateFileEventLog(fileUUID, event, user, details, message) count++ } return err } -func (dbs *SDAdb) updateFileEventLog(fileUUID, event, corrID, user, details, message string) error { +func (dbs *SDAdb) updateFileEventLog(fileUUID, event, user, details, message string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, details, message) VALUES($1, $2, $3, $4, $5, $6);" + const query = ` +INSERT INTO sda.file_event_log(file_id, event, user_id, details, message) +VALUES($1, $2, $3, $4, $5); +` - result, err := db.Exec(query, fileUUID, event, corrID, user, details, message) + result, err := db.Exec(query, fileUUID, event, user, details, message) if err != nil { return err } @@ -268,7 +252,7 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec return nil } -func (dbs *SDAdb) GetFileStatus(corrID string) (string, error) { +func (dbs *SDAdb) GetFileStatus(fileID string) (string, error) { var ( err error count int @@ -276,19 +260,19 @@ func (dbs *SDAdb) GetFileStatus(corrID string) (string, error) { ) for count == 0 || (err != nil && count < RetryTimes) { - status, err = dbs.getFileStatus(corrID) + status, err = dbs.getFileStatus(fileID) count++ } return status, err } -func (dbs *SDAdb) getFileStatus(corrID string) (string, error) { +func (dbs *SDAdb) getFileStatus(fileID string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = "SELECT event from sda.file_event_log WHERE correlation_id = $1 ORDER BY id DESC LIMIT 1;" + const getFileID = "SELECT event from sda.file_event_log WHERE file_id = $1 ORDER BY id DESC LIMIT 1;" var status string - err := db.QueryRow(getFileID, corrID).Scan(&status) + err := db.QueryRow(getFileID, fileID).Scan(&status) if err != nil { return "", err } @@ -372,7 +356,7 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec } // GetArchived retrieves the location and size of archive -func (dbs *SDAdb) GetArchived(corrID string) (string, int, error) { +func (dbs *SDAdb) GetArchived(fileID string) (string, int, error) { var ( filePath string fileSize int @@ -381,13 +365,13 @@ func (dbs *SDAdb) GetArchived(corrID string) (string, int, error) { ) for count == 0 || (err != nil && count < RetryTimes) { - filePath, fileSize, err = dbs.getArchived(corrID) + filePath, fileSize, err = dbs.getArchived(fileID) count++ } return filePath, fileSize, err } -func (dbs *SDAdb) getArchived(corrID string) (string, int, error) { +func (dbs *SDAdb) getArchived(fileID string) (string, int, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -395,7 +379,7 @@ func (dbs *SDAdb) getArchived(corrID string) (string, int, error) { var filePath string var fileSize int - if err := db.QueryRow(query, corrID).Scan(&filePath, &fileSize); err != nil { + if err := db.QueryRow(query, fileID).Scan(&filePath, &fileSize); err != nil { return "", 0, err } @@ -832,11 +816,17 @@ func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*Subm // select all files (that are not part of a dataset) of the user, each one annotated with its latest event const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id -WHERE f.submission_user = $1 and f.submission_file_path LIKE $2 +WHERE f.submission_user = $1 AND ($2 IS NULL OR f.submission_file_root_dir = $2) AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` + pathPrefixArg := sql.NullString{} + if pathPrefix != "" { + pathPrefixArg.Valid = true + pathPrefixArg.String = pathPrefix + } + // nolint:rowserrcheck - rows, err := db.Query(query, userID, fmt.Sprintf("%s%%", pathPrefix)) + rows, err := db.Query(query, userID, pathPrefixArg) if err != nil { return nil, err } @@ -865,53 +855,6 @@ AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` return files, nil } -// get the correlation ID for a user-inbox_path combination -func (dbs *SDAdb) GetCorrID(user, path, accession string) (string, error) { - var ( - corrID string - err error - ) - // 2, 4, 8, 16, 32 seconds between each retry event. - for count := 1; count <= RetryTimes; count++ { - corrID, err = dbs.getCorrID(user, path, accession) - if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return corrID, err -} -func (dbs *SDAdb) getCorrID(user, path, accession string) (string, error) { - dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const query = `SELECT DISTINCT correlation_id FROM sda.file_event_log e -RIGHT JOIN sda.files f ON e.file_id = f.id -WHERE f.submission_file_path = $1 AND f.submission_user = $2 AND COALESCE(f.stable_id, '') = $3;` - - rows, err := db.Query(query, path, user, accession) - if err != nil { - return "", err - } - defer rows.Close() - - var corrID sql.NullString - for rows.Next() { - err := rows.Scan(&corrID) - if err != nil { - return "", err - } - if corrID.Valid { - return corrID.String, nil - } - } - if rows.Err() != nil { - return "", rows.Err() - } - - return "", errors.New("sql: no rows in result set") -} - // list all users with files not yet assigned to a dataset func (dbs *SDAdb) ListActiveUsers() ([]string, error) { dbs.checkAndReconnectIfNeeded() @@ -1338,11 +1281,11 @@ func (dbs *SDAdb) getFileDetailsFromUUID(fileUUID, event string) (FileDetails, e var info FileDetails dbs.checkAndReconnectIfNeeded() - const query = `SELECT f.submission_user, f.submission_file_path, fel.correlation_id + const query = `SELECT f.submission_user, f.submission_file_path from sda.files f join sda.file_event_log fel on f.id = fel.file_id WHERE f.id = $1 and fel.event=$2;` - if err := dbs.DB.QueryRow(query, fileUUID, event).Scan(&info.User, &info.Path, &info.CorrID); err != nil { + if err := dbs.DB.QueryRow(query, fileUUID, event).Scan(&info.User, &info.Path); err != nil { return FileDetails{}, err } From 78416ff91a9eb246f24b9416b4a2e8815d128ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 27 Nov 2025 13:27:05 +0100 Subject: [PATCH 171/184] feat(s3inbox): add back missing delete of file id from cache when upload done --- sda/cmd/s3inbox/proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index ee0d956e0..bfe066dc0 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -169,8 +169,6 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw filePath: filepath, } - log.Infof("fileidentifier: %v, fileId: %s", fileIdentifier, p.fileIDs[fileIdentifier]) - // if this is an upload request if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { // register file in database @@ -252,6 +250,8 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } + + delete(p.fileIDs, fileIdentifier) } // Writing non-200 to the response before the headers propagate the error From 87cba88df6517e74ee5e762333d449a9c19e47f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 27 Nov 2025 13:27:33 +0100 Subject: [PATCH 172/184] feat(database): fix get user files query, cast input param to text --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 3a9085b47..70f84d0eb 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -816,7 +816,7 @@ func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*Subm // select all files (that are not part of a dataset) of the user, each one annotated with its latest event const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id -WHERE f.submission_user = $1 AND ($2 IS NULL OR f.submission_file_root_dir = $2) +WHERE f.submission_user = $1 AND ($2::text IS NULL OR f.submission_file_root_dir = $2::text) AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` pathPrefixArg := sql.NullString{} From 897b45129acc4b80cf7b20be250c8b482ced47cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 27 Nov 2025 13:51:40 +0100 Subject: [PATCH 173/184] feat(api): fix message being produced with empty correlation id when calling file/ingest or file/accession with fileid param --- sda/cmd/api/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 9501e9b24..85384c9c4 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -346,6 +346,7 @@ func ingestFile(c *gin.Context) { // Add file info in the message payload ingest.User = fileDetails.User ingest.FilePath = fileDetails.Path + fileID = c.Query("fileid") case c.Request.ContentLength > 0: // Bind ingest and payload @@ -616,6 +617,7 @@ func setAccession(c *gin.Context) { accession.User = fileDetails.User accession.FilePath = fileDetails.Path accession.DecryptedChecksums = []schema.Checksums{{Type: "sha256", Value: fileDecrChecksum}} + fileID = c.Query("fileid") case c.Request.ContentLength > 0: if err = c.BindJSON(&accession); err != nil { From 7e65ffa90bcd849c40f426b1cdf4c969fd4168a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 27 Nov 2025 14:11:52 +0100 Subject: [PATCH 174/184] feat(integration tests): update sda integration tests to work with database schema changes --- .github/integration/tests/sda/21_cancel_test.sh | 2 +- .github/integration/tests/sda/22_error_test.sh | 6 +++--- .github/integration/tests/sda/31_cancel_test2.sh | 8 ++++---- .github/integration/tests/sda/92_handle_file_errors.sh | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/integration/tests/sda/21_cancel_test.sh b/.github/integration/tests/sda/21_cancel_test.sh index c7482bdba..f67dffe08 100644 --- a/.github/integration/tests/sda/21_cancel_test.sh +++ b/.github/integration/tests/sda/21_cancel_test.sh @@ -55,7 +55,7 @@ curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ -d "$cancel_body" | jq # check database to verify file status -if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1")" != "disabled" ]; then +if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$CORRID' order by id DESC LIMIT 1")" != "disabled" ]; then echo "canceling file failed" exit 1 fi diff --git a/.github/integration/tests/sda/22_error_test.sh b/.github/integration/tests/sda/22_error_test.sh index ea79144e8..5ea7ae946 100644 --- a/.github/integration/tests/sda/22_error_test.sh +++ b/.github/integration/tests/sda/22_error_test.sh @@ -71,7 +71,7 @@ curl -k -u guest:guest "$URI/api/exchanges/sda/sda/publish" \ -d "$ingest_body" | jq # check database to verify file status -until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE correlation_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do echo "waiting for file error to be logged by ingest" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then @@ -83,7 +83,7 @@ done ## give the file a non existing archive path psql -U postgres -h postgres -d sda -Atq -c "UPDATE sda.files SET archive_file_path = '$CORRID', header = '637279707434676801000000010000006c00000000000000' WHERE id = '$CORRID';" -psql -U postgres -h postgres -d sda -Atq -c "INSERT INTO sda.file_event_log(file_id, correlation_id, event) VALUES('$CORRID', '$CORRID', 'archived');" +psql -U postgres -h postgres -d sda -Atq -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$CORRID', 'archived');" encrypted_checksums=$( jq -c -n \ @@ -119,7 +119,7 @@ curl -k -u guest:guest "$URI/api/exchanges/sda/sda/publish" \ # check database to verify file status RETRY_TIMES=0 -until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE correlation_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id = '$CORRID' ORDER BY ID DESC LIMIT 1;")" = "error" ]; do echo "waiting for file error to be logged by verify" date RETRY_TIMES=$((RETRY_TIMES + 1)) diff --git a/.github/integration/tests/sda/31_cancel_test2.sh b/.github/integration/tests/sda/31_cancel_test2.sh index 0095a0687..8b6101c34 100644 --- a/.github/integration/tests/sda/31_cancel_test2.sh +++ b/.github/integration/tests/sda/31_cancel_test2.sh @@ -7,13 +7,13 @@ ENC_SHA=$(sha256sum NA12878.bam.c4gh | cut -d' ' -f 1) ENC_MD5=$(md5sum NA12878.bam.c4gh | cut -d' ' -f 1) ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';") +FILEID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';") properties=$( jq -c -n \ --argjson delivery_mode 2 \ - --arg correlation_id "$CORRID" \ + --arg correlation_id "$FILEID" \ --arg content_encoding UTF-8 \ --arg content_type application/json \ '$ARGS.named' @@ -52,7 +52,7 @@ curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ # check database to verify file status RETRY_TIMES=0 -until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1;")" = "disabled" ]; do +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$FILEID' order by id DESC LIMIT 1;")" = "disabled" ]; do echo "canceling file failed" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then @@ -132,7 +132,7 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ -d "$accession_body" | jq RETRY_TIMES=0 -until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where correlation_id = '$CORRID' order by id DESC LIMIT 1")" = "ready" ]; do +until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_event_log where file_id = '$FILEID' order by id DESC LIMIT 1")" = "ready" ]; do echo "waiting for re-ingested file to become ready" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then diff --git a/.github/integration/tests/sda/92_handle_file_errors.sh b/.github/integration/tests/sda/92_handle_file_errors.sh index 7316d1bac..74685836d 100644 --- a/.github/integration/tests/sda/92_handle_file_errors.sh +++ b/.github/integration/tests/sda/92_handle_file_errors.sh @@ -81,8 +81,8 @@ missing_file_payload=$( '$ARGS.named|@base64' ) -FILEID=$(psql -U postgres -h postgres -d sda -At -c "SELECT DISTINCT(file_id) FROM sda.file_event_log WHERE correlation_id = '$CORRID';") -psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$FILEID', 'uploaded', '$CORRID', 'test@dummy.org', '{\"uploaded\": \"message\"}');" +FILEID=$(psql -U postgres -h postgres -d sda -At -c "SELECT DISTINCT(file_id) FROM sda.file_event_log WHERE file_id = '$CORRID';") +psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, user_id, message) VALUES('$FILEID', 'uploaded', 'test@dummy.org', '{\"uploaded\": \"message\"}');" properties=$( jq -c -n \ From 203d702263964405c3f5b54aaef7f1251b3a89f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 13:26:28 +0100 Subject: [PATCH 175/184] feat(database): remove addition of submission_file_root_dir column, instead improve GetUserFiles query(also improve getFileIDByUserPathAndStatus query), index on sda.files on submission_user and sumission_file_path --- postgresql/initdb.d/01_main.sql | 5 ++- ...add_index_on_files_and_file_event_log.sql} | 9 ++--- ...precate_files_event_log_correlation_id.sql | 1 + sda/internal/database/db_functions.go | 35 ++++++++++++------- 4 files changed, 29 insertions(+), 21 deletions(-) rename postgresql/migratedb.d/{19_add_indexes_and_new_column_on_files.sql => 19_add_index_on_files_and_file_event_log.sql} (71%) diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index c007e904c..572b8aa8f 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -58,7 +58,6 @@ CREATE TABLE files ( submission_user TEXT, submission_file_path TEXT DEFAULT '' NOT NULL, - submission_file_root_dir TEXT GENERATED ALWAYS AS (split_part(submission_file_path, '/', 1)) STORED, submission_file_size BIGINT, archive_file_path TEXT DEFAULT '' NOT NULL, @@ -79,7 +78,7 @@ CREATE TABLE files ( CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user) ); -- Add indexes to the files table -CREATE INDEX files_submission_user_submission_file_root_dir_idx ON sda.files(submission_user, submission_file_root_dir); +CREATE INDEX files_submission_user_submission_file_path_idx ON sda.files(submission_user, submission_file_path); -- The user info is used by auth to be able to link users to their name and email CREATE TABLE userinfo ( @@ -169,7 +168,7 @@ CREATE TABLE file_event_log ( error TEXT ); -- Add indexes to the file_event_log table -CREATE INDEX file_event_log_file_id_started_at_idx ON sda.file_event_log(file_id, started_at); +CREATE INDEX file_event_log_file_id_started_at_idx ON file_event_log(file_id, started_at); -- This table is used to define events for dataset event logging. CREATE TABLE dataset_events ( diff --git a/postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql b/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql similarity index 71% rename from postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql rename to postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql index 7825f2688..d1c378576 100644 --- a/postgresql/migratedb.d/19_add_indexes_and_new_column_on_files.sql +++ b/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql @@ -12,13 +12,10 @@ BEGIN RAISE NOTICE 'Changes: %', changes; INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + CREATE INDEX file_event_log_file_id_started_at_idx ON sda.file_event_log(file_id, started_at); - ALTER TABLE sda.files - ADD COLUMN submission_file_root_dir TEXT GENERATED ALWAYS AS (split_part(submission_file_path, '/', 1)) STORED; - - - CREATE INDEX files_submission_user_submission_file_root_dir_idx - ON sda.files(submission_user, submission_file_root_dir); + CREATE INDEX files_submission_user_submission_file_path_idx + ON sda.files(submission_user, submission_file_path); ELSE RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; diff --git a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql index bd90b2c29..21e77bc7f 100644 --- a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql +++ b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql @@ -72,6 +72,7 @@ BEGIN END; $register_file$ LANGUAGE plpgsql; + -- Drop the correlation_id column from sda.file_event_log ALTER TABLE sda.file_event_log DROP COLUMN correlation_id; diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 70f84d0eb..948f9650d 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -103,12 +103,17 @@ func (dbs *SDAdb) getFileIDByUserPathAndStatus(submissionUser, filePath, status dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = `SELECT id from sda.files -WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null -AND EXISTS (SELECT 1 FROM -(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id -WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) -AS subquery WHERE event = $3);` + const getFileID = ` +SELECT id_and_event.id +FROM ( + SELECT DISTINCT ON (f.id) f.id, fel.event FROM sda.files AS f + LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id + WHERE f.submission_user = $1 + AND f.submission_file_path = $2 + AND f.stable_id IS null + ORDER BY f.id, fel.started_at DESC LIMIT 1 + ) AS id_and_event +WHERE id_and_event.event = $3;` var fileID string err := db.QueryRow(getFileID, submissionUser, filePath, status).Scan(&fileID) @@ -814,20 +819,26 @@ func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*Subm db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f -LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id -WHERE f.submission_user = $1 AND ($2::text IS NULL OR f.submission_file_root_dir = $2::text) -AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` - + const query = ` +SELECT DISTINCT ON (f.id) f.id, f.submission_file_path, f.stable_id, fel.event, f.created_at FROM sda.files AS f + LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id + LEFT JOIN sda.file_dataset AS fd ON fd.file_id = f.id +WHERE f.submission_user = $1 AND ($2::TEXT IS NULL OR substr(f.submission_file_path, 1, $3) = $2::TEXT) + AND fd.file_id IS NULL +ORDER BY f.id, fel.started_at DESC;` + + pathPrefixLen := 1 pathPrefixArg := sql.NullString{} if pathPrefix != "" { + pathPrefixLen = len(pathPrefix) pathPrefixArg.Valid = true pathPrefixArg.String = pathPrefix } // nolint:rowserrcheck - rows, err := db.Query(query, userID, pathPrefixArg) + rows, err := db.Query(query, userID, pathPrefixArg, pathPrefixLen) if err != nil { + log.Errorf("Error querying user files: %v", err) return nil, err } defer rows.Close() From 1d85f5324f5f5ffe0e48fa1764b8f8a8f6f7cc51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 15:18:10 +0100 Subject: [PATCH 176/184] feat(unit tests): fix unit tests to work with database changes, also update getFileIDByUserPathAndStatus to return id if stable id is set --- sda/cmd/api/api_test.go | 119 +++++------ sda/cmd/ingest/ingest.go | 3 + sda/cmd/ingest/ingest_test.go | 89 ++++---- sda/cmd/rotatekey/rotatekey_test.go | 12 +- sda/cmd/s3inbox/proxy_test.go | 8 +- sda/cmd/sync/sync_test.go | 2 +- sda/internal/database/db_functions.go | 6 +- sda/internal/database/db_functions_test.go | 229 ++++++--------------- 8 files changed, 183 insertions(+), 285 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index cd8660b1b..f6f0aaabd 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -26,7 +26,6 @@ import ( "github.com/casbin/casbin/v2" "github.com/casbin/casbin/v2/model" "github.com/gin-gonic/gin" - "github.com/google/uuid" _ "github.com/lib/pq" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/streaming" @@ -299,9 +298,9 @@ type TestSuite struct { } func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, hash.Hash) { - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") encSha := sha256.New() @@ -323,7 +322,7 @@ func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Verified") - err = Conf.API.DB.UpdateFileEventLog(fileID, "verified", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "verified", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") return fileID, decSha @@ -670,12 +669,11 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a file and make sure it is listed file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFile(file1, s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, file1, s.User) assert.NoError(s.T(), err, "failed to register file in database") - corrID := uuid.New().String() latestStatus := "uploaded" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, s.User, "{}", "{}") assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) @@ -695,7 +693,7 @@ func (s *TestSuite) TestAPIGetFiles() { s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) } latestStatus = "ready" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, s.User, "{}", "{}") assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) @@ -715,7 +713,7 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a second file and make sure it is listed file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", s.User) - _, err = Conf.API.DB.RegisterFile(file2, s.User) + _, err = Conf.API.DB.RegisterFile(nil, file2, s.User) assert.NoError(s.T(), err, "failed to register file in database") resp, err = client.Do(req) @@ -749,12 +747,12 @@ func (s *TestSuite) TestAPIGetFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -1085,9 +1083,9 @@ func (s *TestSuite) TestIngestFile_WithPayload() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) @@ -1144,9 +1142,9 @@ func (s *TestSuite) TestIngestFile_WithPayload_NoUser() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) @@ -1178,9 +1176,9 @@ func (s *TestSuite) TestIngestFile_WithPayload_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) @@ -1211,9 +1209,9 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) @@ -1245,9 +1243,9 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { func (s *TestSuite) TestIngestFile_WithFileID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err) - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) gin.SetMode(gin.ReleaseMode) @@ -1290,9 +1288,9 @@ func (s *TestSuite) TestIngestFile_WithFileID() { func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err) - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) gin.SetMode(gin.ReleaseMode) @@ -1316,9 +1314,9 @@ func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err) - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) gin.SetMode(gin.ReleaseMode) @@ -1653,10 +1651,10 @@ func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(s.T(), err, "failed to update satus of file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") + assert.NoError(s.T(), err, "failed to update status of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) @@ -1680,7 +1678,10 @@ func (s *TestSuite) TestCreateDataset() { assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file accession", err) + + err = Conf.API.DB.UpdateFileEventLog(fileID, "ready", user, "{}", "{}") + assert.NoError(s.T(), err, "got (%v) when setting file status ready", err) gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) @@ -1732,9 +1733,9 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() @@ -1764,7 +1765,7 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) - err = Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "ready", "finalize", "{}", "{}") assert.NoError(s.T(), err, "got (%v) when marking file as ready", err) gin.SetMode(gin.ReleaseMode) @@ -1854,9 +1855,9 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() @@ -1912,12 +1913,12 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { func (s *TestSuite) TestReleaseDataset() { user := "TestReleaseDataset" for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -2045,12 +2046,12 @@ func (s *TestSuite) TestReleaseDataset_DeprecatedDataset() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -2104,12 +2105,12 @@ func (s *TestSuite) TestListActiveUsers() { testUsers := []string{"User-A", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -2163,12 +2164,12 @@ func (s *TestSuite) TestListUserFiles() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -2229,12 +2230,12 @@ func (s *TestSuite) TestListUserFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { s.FailNow("failed to update satus of file in database") } @@ -2490,7 +2491,7 @@ func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { func (s *TestSuite) TestListDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") if err != nil { s.FailNow("failed to register file in database") } @@ -2548,7 +2549,7 @@ func (s *TestSuite) TestListDatasets() { func (s *TestSuite) TestListUserDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2605,7 +2606,7 @@ func (s *TestSuite) TestListUserDatasets() { func (s *TestSuite) TestListDatasetsAsUser() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) if err != nil { s.FailNow("failed to register file in database") } @@ -2664,12 +2665,12 @@ func (s *TestSuite) TestReVerifyFile() { user := "TestReVerify" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerify-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) if err != nil { s.FailNow("failed to register file in database") } - if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { + if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}"); err != nil { s.FailNow("failed to update satus of file in database") } encSha := sha256.New() @@ -2704,7 +2705,7 @@ func (s *TestSuite) TestReVerifyFile() { if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } - if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { + if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", "finalize", "{}", "{}"); err != nil { s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } @@ -2766,12 +2767,12 @@ func (s *TestSuite) TestReVerifyDataset() { user := "TestReVerifyDataset" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerifyDataset-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFile(filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) if err != nil { s.FailNow("failed to register file in database") } - if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { + if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}"); err != nil { s.FailNow("failed to update satus of file in database") } encSha := sha256.New() @@ -2806,7 +2807,7 @@ func (s *TestSuite) TestReVerifyDataset() { if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } - if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { + if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", "finalize", "{}", "{}"); err != nil { s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } @@ -2886,9 +2887,9 @@ func (s *TestSuite) TestDownloadFile() { defer ts.Close() // Register the file in the database - fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.GoodC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, filepath.Base(s.GoodC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") // Mock request to download the file @@ -2973,9 +2974,9 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { // Register a file in the database (but don't create the actual file) filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, filePath, s.User) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") // Mock request to download the file @@ -3005,9 +3006,9 @@ func (s *TestSuite) TestDownloadFile_badC4ghFile() { defer ts.Close() // Register a file in the database (but don't create the actual file) - fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.BadC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, filepath.Base(s.BadC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") - err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") // Mock request to download the file diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 772dfceef..f8f834245 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -206,6 +206,9 @@ func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) st m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to update event log for file with id : %s", fileID) + if strings.Contains(err.Error(), "sql: no rows in result set") { + return "reject" + } return "nack" } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index ead55be34..a5980352e 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -385,11 +385,10 @@ func (ts *TestSuite) TestCancelFile() { // prepare the DB entries userName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) - fileID, err := ts.ingest.DB.RegisterFile(file1, userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -399,17 +398,16 @@ func (ts *TestSuite) TestCancelFile() { User: userName, } - assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(fileID, message)) } func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // prepare the DB entries userName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", userName) - fileID, err := ts.ingest.DB.RegisterFile(file1, userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -419,17 +417,16 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { User: userName, } - assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.New().String(), message)) + assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.NewString(), message)) } // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -439,15 +436,14 @@ func (ts *TestSuite) TestIngestFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) } func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -457,10 +453,10 @@ func (ts *TestSuite) TestIngestFile_secondTime() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) // file is already in `archived` state - assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(fileID, message)) } func (ts *TestSuite) TestIngestFile_unknownInboxType() { message := schema.IngestionTrigger{ @@ -473,11 +469,10 @@ func (ts *TestSuite) TestIngestFile_unknownInboxType() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -487,21 +482,20 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -511,9 +505,9 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -555,7 +549,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { crypt4GHWriter.Close() // reingestion should work - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) // DB should have the new checksum var dbChecksum string @@ -568,11 +562,10 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -582,7 +575,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) // fake file verification sha256hash := sha256.New() @@ -595,15 +588,14 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { ts.Fail("failed to mark file as verified") } - assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -613,7 +605,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) // fake file verification sha256hash := sha256.New() @@ -626,19 +618,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { ts.Fail("failed to mark file as verified") } - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") - corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -648,7 +639,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) var firstDbChecksum string const q1 = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" @@ -667,7 +658,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { ts.Fail("failed to mark file as verified") } - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } @@ -709,7 +700,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { crypt4GHWriter.Close() // reingestion should work - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) // DB should have the new checksum var dbChecksum string @@ -724,16 +715,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_missingFile() { // prepare the DB entries - corrID := uuid.New().String() + basepath := filepath.Dir(ts.filePath) + newFileID := uuid.NewString() + message := schema.IngestionTrigger{ Type: "ingest", FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(newFileID, message)) } func (ts *TestSuite) TestDetectMisingC4GHKeys() { viper.Set("c4gh.privateKeys", "") diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index 014fc5e05..e4c8a4399 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -158,7 +158,6 @@ func TestMain(m *testing.M) { type TestSuite struct { suite.Suite app RotateKey - corrID string fileID string privateKeyList []*[32]byte } @@ -174,7 +173,6 @@ func TestRotateKeyTestSuite(t *testing.T) { func (ts *TestSuite) SetupSuite() { ts.app.Conf = &config.Config{} ts.app.Conf.Broker.SchemasPath = "../../schemas/isolated" - ts.corrID = uuid.New().String() var err error ts.app.DB, err = database.NewSDAdb(database.DBConf{ Host: "localhost", @@ -214,12 +212,12 @@ func (ts *TestSuite) SetupSuite() { ts.app.Conf.RotateKey.PublicKey = &publicKey - ts.fileID, err = ts.app.DB.RegisterFile("rotate-key-test/data.c4gh", "tester_example.org") + ts.fileID, err = ts.app.DB.RegisterFile(nil, "rotate-key-test/data.c4gh", "tester_example.org") if err != nil { ts.FailNow("Failed to register file in DB") } for _, status := range []string{"uploaded", "archived", "verified"} { - if err = ts.app.DB.UpdateFileEventLog(ts.fileID, status, ts.corrID, "tester_example.org", "{}", "{}"); err != nil { + if err = ts.app.DB.UpdateFileEventLog(ts.fileID, status, "tester_example.org", "{}", "{}"); err != nil { ts.FailNow("Failed to set status of file in DB") } } @@ -290,7 +288,6 @@ func (s *server) ReencryptHeader(ctx context.Context, req *re.ReencryptRequest) } func (ts *TestSuite) TestReEncryptHeader() { - fileID := ts.corrID for _, test := range []struct { corrID string @@ -305,16 +302,15 @@ func (ts *TestSuite) TestReEncryptHeader() { expectedError: nil, expectedMgs: "", expectedRes: "ack", - corrID: ts.corrID, fileID: ts.fileID, }, { testName: "un-ingested file", expectedError: errors.New("sql: no rows in result set"), - expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", fileID), + expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", ts.fileID), expectedRes: "ackSendToError", corrID: uuid.New().String(), - fileID: fileID, + fileID: ts.fileID, }, } { ts.T().Run(test.testName, func(t *testing.T) { diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 13e901b58..e38be3ac1 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -630,7 +630,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB() { p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db - fileID, err := db.RegisterFile("/dummy/file", "test-user") + fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) @@ -653,7 +653,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db - fileID, err := db.RegisterFile("/dummy/file", "test-user") + fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) @@ -672,7 +672,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db - fileID, err := db.RegisterFile("/dummy/file", "test-user") + fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) @@ -699,7 +699,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db - fileID, err := db.RegisterFile("/test/new_file", "test-user") + fileID, err := db.RegisterFile(nil, "/test/new_file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index fd31473db..c3c10d2f6 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -159,7 +159,7 @@ func (s *SyncTest) TestBuildSyncDatasetJSON() { db, err = database.NewSDAdb(conf.Database) assert.NoError(s.T(), err) - fileID, err := db.RegisterFile("dummy.user/test/file1.c4gh", "dummy.user") + fileID, err := db.RegisterFile(nil, "dummy.user/test/file1.c4gh", "dummy.user") assert.NoError(s.T(), err, "failed to register file in database") err = db.SetAccessionID("ed6af454-d910-49e3-8cda-488a6f246e67", fileID) assert.NoError(s.T(), err) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 948f9650d..84b731fec 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -110,7 +110,6 @@ FROM ( LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id WHERE f.submission_user = $1 AND f.submission_file_path = $2 - AND f.stable_id IS null ORDER BY f.id, fel.started_at DESC LIMIT 1 ) AS id_and_event WHERE id_and_event.event = $3;` @@ -150,6 +149,11 @@ VALUES($1, $2, $3, $4, $5); result, err := db.Exec(query, fileUUID, event, user, details, message) if err != nil { + // 23503 error code == foreign_key_violation, meaning the files row does not exits + // http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html + if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "23503" { + return sql.ErrNoRows + } return err } if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 60da6516d..877f06fc0 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -20,7 +20,7 @@ func (suite *DatabaseTests) TestRegisterFile() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") // check that the returning fileID is a uuid @@ -42,19 +42,20 @@ func (suite *DatabaseTests) TestRegisterFile() { db.Close() } -func (suite *DatabaseTests) TestGetFileID() { +func (suite *DatabaseTests) TestRegisterFileWithID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/file3.c4gh", "testuser") + insertedFileID := uuid.New().String() + fileID, err := db.RegisterFile(&insertedFileID, "/testuser/file3.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - corrID := uuid.New().String() - err = db.UpdateFileEventLog(fileID, "uploaded", corrID, "testuser", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "testuser", "{}", "{}") assert.NoError(suite.T(), err, "failed to update file status") - fID, err := db.GetFileID(corrID) + fID, err := db.GetFileIDByUserPathAndStatus("testuser", "/testuser/file3.c4gh", "uploaded") assert.NoError(suite.T(), err, "GetFileId failed") + assert.Equal(suite.T(), insertedFileID, fileID) assert.Equal(suite.T(), fileID, fID) db.Close() @@ -65,16 +66,15 @@ func (suite *DatabaseTests) TestUpdateFileEventLog() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/file4.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file4.c4gh", "testuser") assert.Nil(suite.T(), err, "failed to register file in database") - corrID := uuid.New().String() // Attempt to mark a file that doesn't exist as uploaded - err = db.UpdateFileEventLog("00000000-0000-0000-0000-000000000000", "uploaded", corrID, "testuser", "{}", "{}") + err = db.UpdateFileEventLog("00000000-0000-0000-0000-000000000000", "uploaded", "testuser", "{}", "{}") assert.NotNil(suite.T(), err, "Unknown file could be marked as uploaded in database") // mark file as uploaded - err = db.UpdateFileEventLog(fileID, "uploaded", corrID, "testuser", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "testuser", "{}", "{}") assert.NoError(suite.T(), err, "failed to set file as uploaded in database") exists := false @@ -91,7 +91,7 @@ func (suite *DatabaseTests) TestStoreHeader() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestStoreHeader.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestStoreHeader.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) @@ -109,7 +109,7 @@ func (suite *DatabaseTests) TestRotateHeaderKey() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // Register a new key and a new file - fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.addKeyHash("someKeyHash", "this is a test key") assert.NoError(suite.T(), err, "failed to register key in database") @@ -157,7 +157,7 @@ func (suite *DatabaseTests) TestSetArchived() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetArchived.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestSetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())} @@ -178,14 +178,13 @@ func (suite *DatabaseTests) TestGetFileStatus() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetFileStatus.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetFileStatus.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - corrID := uuid.New().String() - err = db.UpdateFileEventLog(fileID, "downloaded", corrID, "testuser", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "downloaded", "testuser", "{}", "{}") assert.NoError(suite.T(), err, "failed to set file as downloaded in database") - status, err := db.GetFileStatus(corrID) + status, err := db.GetFileStatus(fileID) assert.NoError(suite.T(), err, "failed to get file status") assert.Equal(suite.T(), "downloaded", status) @@ -197,7 +196,7 @@ func (suite *DatabaseTests) TestGetHeader() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetHeader.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetHeader.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) @@ -215,7 +214,7 @@ func (suite *DatabaseTests) TestSetVerified() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetVerified.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestSetVerified.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948, fmt.Sprintf("%x", sha256.New())} @@ -233,7 +232,7 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetArchived.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -256,7 +255,7 @@ func (suite *DatabaseTests) TestSetAccessionID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -276,7 +275,7 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestCheckAccessionIDExists.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestCheckAccessionIDExists.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -304,7 +303,7 @@ func (suite *DatabaseTests) TestGetAccessionID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -328,7 +327,7 @@ func (suite *DatabaseTests) TestGetAccessionID_wrongFileID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -359,7 +358,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetFileInfo.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetFileInfo.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") encSha := sha256.New() @@ -393,7 +392,7 @@ func (suite *DatabaseTests) TestMapFilesToDataset() { accessions := []string{} for i := 1; i < 12; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestMapFilesToDataset-%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestMapFilesToDataset-%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-%d", i), fileID) @@ -432,7 +431,7 @@ func (suite *DatabaseTests) TestGetInboxPath() { accessions := []string{} for i := 0; i < 5; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) @@ -456,7 +455,7 @@ func (suite *DatabaseTests) TestUpdateDatasetEvent() { accessions := []string{} for i := 0; i < 5; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) @@ -490,7 +489,7 @@ func (suite *DatabaseTests) TestGetHeaderForStableID() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetHeaderForStableID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetHeaderForStableID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte("HEADER"), fileID) @@ -512,7 +511,7 @@ func (suite *DatabaseTests) TestGetSyncData() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile("/testuser/TestGetGetSyncData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetGetSyncData.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) @@ -543,7 +542,7 @@ func (suite *DatabaseTests) TestCheckIfDatasetExists() { accessions := []string{} for i := 0; i <= 3; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/testuser/TestCheckIfDatasetExists-%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestCheckIfDatasetExists-%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("accession-%d", i), fileID) @@ -576,7 +575,7 @@ func (suite *DatabaseTests) TestGetArchivePath() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/TestGetArchivePath-001.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetArchivePath-001.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New()) @@ -607,13 +606,13 @@ func (suite *DatabaseTests) TestGetUserFiles() { sub = "submission_b" } - fileID, err := db.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) + fileID, err := db.RegisterFile(nil, fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) assert.NoError(suite.T(), err, "failed to register file in database") - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, testUser, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") err = db.SetAccessionID(fmt.Sprintf("stableID-00%d", i), fileID) assert.NoError(suite.T(), err, "failed to update satus of file in database") - err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "ready", testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } filelist, err := db.GetUserFiles("unknownuser", "", true) @@ -636,25 +635,6 @@ func (suite *DatabaseTests) TestGetUserFiles() { db.Close() } -func (suite *DatabaseTests) TestGetCorrID() { - db, err := NewSDAdb(suite.dbConf) - assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - - filePath := "/testuser/file10.c4gh" - user := "testuser" - - fileID, err := db.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") - - corrID, err := db.GetCorrID(user, filePath, "") - assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") - assert.Equal(suite.T(), fileID, corrID) - - db.Close() -} - func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) @@ -662,11 +642,11 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { filePath := "/testuser/file10.c4gh" user := "testuser" - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("failed to register file in database") } - if err := db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}"); err != nil { + if err := db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}"); err != nil { suite.FailNow("failed to update satus of file in database") } @@ -675,66 +655,20 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { if err := db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as archived") } - if err := db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}"); err != nil { + if err := db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}"); err != nil { suite.FailNow("failed to update satus of file in database") } if err = db.SetAccessionID("stableID", fileID); err != nil { suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) } - fileID2, err := db.RegisterFile(filePath, user) + fileID2, err := db.RegisterFile(nil, filePath, user) assert.NoError(suite.T(), err, "failed to register file in database") - if err := db.UpdateFileEventLog(fileID2, "uploaded", fileID2, user, "{}", "{}"); err != nil { + if err := db.UpdateFileEventLog(fileID2, "uploaded", user, "{}", "{}"); err != nil { suite.FailNow("failed to update satus of file in database") } assert.NotEqual(suite.T(), fileID, fileID2) - corrID, err := db.GetCorrID(user, filePath, "") - assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") - assert.Equal(suite.T(), fileID2, corrID) - - db.Close() -} - -func (suite *DatabaseTests) TestGetCorrID_wrongFilePath() { - db, err := NewSDAdb(suite.dbConf) - assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - - filePath := "/testuser/file10.c4gh" - user := "testuser" - - fileID, err := db.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update status of file in database") - - corrID, err := db.GetCorrID(user, "/testuser/file20.c4gh", "") - assert.EqualError(suite.T(), err, "sql: no rows in result set") - assert.Equal(suite.T(), "", corrID) - - db.Close() -} - -func (suite *DatabaseTests) TestGetCorrID_fileWithAccessionID() { - db, err := NewSDAdb(suite.dbConf) - assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - - filePath := "/testuser/file10.c4gh" - user := "testuser" - - fileID, err := db.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") - if err := db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { - suite.FailNow("failed to update satus of file in database") - } - if err = db.SetAccessionID("stableID", fileID); err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) - } - - corrID, err := db.GetCorrID(user, filePath, "stableID") - assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") - assert.Equal(suite.T(), fileID, corrID) - db.Close() } @@ -747,21 +681,15 @@ func (suite *DatabaseTests) TestListActiveUsers() { for _, user := range testUsers { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i) - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } - corrID, err := db.GetCorrID(user, filePath, "") - if err != nil { - suite.FailNow("Failed to get CorrID for file") - } - assert.Equal(suite.T(), fileID, corrID) - checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999, fmt.Sprintf("%x", sha256.New())} err = db.SetArchived(fileInfo, fileID) @@ -808,21 +736,15 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(filePath, "User-Q") + fileID, err := db.RegisterFile(nil, filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-Q", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "User-Q", "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } - corrID, err := db.GetCorrID("User-Q", filePath, "") - if err != nil { - suite.FailNow("Failed to get CorrID for file") - } - assert.Equal(suite.T(), fileID, corrID) - checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{ fmt.Sprintf("%x", sha256.New().Sum(nil)), @@ -969,7 +891,7 @@ func (suite *DatabaseTests) TestSetKeyHash() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") // Test that the key hash can be set in the files table @@ -993,7 +915,7 @@ func (suite *DatabaseTests) TestSetKeyHash_wrongHash() { keyDescription := "this is a test hash" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile("/testuser/file2.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file2.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") // Ensure failure if a non existing hash is used @@ -1012,7 +934,7 @@ func (suite *DatabaseTests) TestGetKeyHash() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) assert.NoError(suite.T(), err, "failed to set key hash in database") @@ -1033,7 +955,7 @@ func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile("/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) assert.NoError(suite.T(), err, "failed to set key hash in database") @@ -1093,21 +1015,15 @@ func (suite *DatabaseTests) TestListDatasets() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(filePath, "User-Q") + fileID, err := db.RegisterFile(nil, filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-Q", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "User-Q", "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } - corrID, err := db.GetCorrID("User-Q", filePath, "") - if err != nil { - suite.FailNow("Failed to get CorrID for file") - } - assert.Equal(suite.T(), fileID, corrID) - checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{ fmt.Sprintf("%x", sha256.New().Sum(nil)), @@ -1179,21 +1095,15 @@ func (suite *DatabaseTests) TestListUserDatasets() { user := "User-Q" for i := 0; i < 6; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i) - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } - corrID, err := db.GetCorrID(user, filePath, "") - if err != nil { - suite.FailNow("Failed to get CorrID for file") - } - assert.Equal(suite.T(), fileID, corrID) - checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{ fmt.Sprintf("%x", sha256.New().Sum(nil)), @@ -1236,7 +1146,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { suite.FailNow("failed to update dataset event") } - fileID, err := db.RegisterFile("filePath", "user") + fileID, err := db.RegisterFile(nil, "filePath", "user") if err != nil { suite.FailNow("Failed to register file") } @@ -1324,7 +1234,7 @@ func (suite *DatabaseTests) TestGetReVerificationData() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1364,7 +1274,7 @@ func (suite *DatabaseTests) TestGetReVerificationDataFromFileID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1400,7 +1310,7 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1441,7 +1351,7 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile("/testuser/TestGetDecryptedChecksum.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/testuser/TestGetDecryptedChecksum.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1481,21 +1391,15 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetDsatasetFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(filePath, "User-Q") + fileID, err := db.RegisterFile(nil, filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-Q", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "User-Q", "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } - corrID, err := db.GetCorrID("User-Q", filePath, "") - if err != nil { - suite.FailNow("Failed to get CorrID for file") - } - assert.Equal(suite.T(), fileID, corrID) - checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{ fmt.Sprintf("%x", sha256.New().Sum(nil)), @@ -1540,11 +1444,11 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { user := "UserX" filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("Failed to register file") } - err = db.UpdateFileEventLog(fileID, "uploaded", fileID, "User-z", "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", "User-z", "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } @@ -1552,7 +1456,7 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { assert.NoError(suite.T(), err) assert.Equal(suite.T(), path, filePath) - err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}") assert.NoError(suite.T(), err) _, err = db.getInboxFilePathFromID(user, fileID) assert.Error(suite.T(), err) @@ -1565,7 +1469,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { user := "UserX" filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -1579,7 +1483,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { assert.Equal(suite.T(), fileID, fileID2) // update the status of the file - err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}") if err != nil { suite.FailNow("Failed to update file event log") } @@ -1604,14 +1508,13 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { // Register a file to get a valid UUID filePath := "/dummy_user.org/Dummy_folder/dummyfile.c4gh" user := "dummy@user.org" - fileID, err := db.RegisterFile(filePath, user) + fileID, err := db.RegisterFile(nil, filePath, user) if err != nil { suite.FailNow("failed to register file in database") } // Update event log to ensure correlation ID is set - correlationID := "b7e2c1a4-5f3b-4c8e-9d2a-7f6e1b2c3d4e" - err = db.UpdateFileEventLog(fileID, "uploaded", correlationID, user, "{}", "{}") + err = db.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") if err != nil { suite.FailNow("failed to update file event log") } @@ -1620,7 +1523,6 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { assert.NoError(suite.T(), err, "failed to get user and path from UUID") assert.Equal(suite.T(), user, infoFile.User) assert.Equal(suite.T(), filePath, infoFile.Path) - assert.Equal(suite.T(), correlationID, infoFile.CorrID) db.Close() } @@ -1634,7 +1536,6 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUID_NotFound() { assert.Error(suite.T(), err, "expected error for non-existent UUID") assert.Empty(suite.T(), infoFile.User) assert.Empty(suite.T(), infoFile.Path) - assert.Empty(suite.T(), infoFile.CorrID) db.Close() } From 56e49cff0fe8e35e374dfae659e3458732b49836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 15:21:34 +0100 Subject: [PATCH 177/184] feat(postgres): remove sda.prefix on index creation in 01_main.sql file --- postgresql/initdb.d/01_main.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index 572b8aa8f..e0b710577 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -78,7 +78,7 @@ CREATE TABLE files ( CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user) ); -- Add indexes to the files table -CREATE INDEX files_submission_user_submission_file_path_idx ON sda.files(submission_user, submission_file_path); +CREATE INDEX files_submission_user_submission_file_path_idx ON files(submission_user, submission_file_path); -- The user info is used by auth to be able to link users to their name and email CREATE TABLE userinfo ( From 0721f93503062eda41cdc7b015f4ed39c1525407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 15:41:41 +0100 Subject: [PATCH 178/184] feat(linting): fix golangci-lint issues accorrding to golangci.yml --- sda/cmd/api/api_test.go | 40 ++++++++++---------- sda/cmd/auth/info_test.go | 2 +- sda/cmd/auth/jwt_test.go | 2 +- sda/cmd/ingest/ingest.go | 2 - sda/cmd/ingest/ingest_test.go | 2 +- sda/cmd/notify/notify_test.go | 2 +- sda/cmd/reencrypt/reencrypt_test.go | 2 +- sda/cmd/rotatekey/rotatekey.go | 2 +- sda/cmd/rotatekey/rotatekey_test.go | 3 +- sda/cmd/s3inbox/proxy_test.go | 6 +-- sda/cmd/s3inbox/s3inbox_test.go | 6 +-- sda/cmd/syncapi/syncapi_test.go | 2 +- sda/internal/broker/broker.go | 2 +- sda/internal/broker/broker_test.go | 4 +- sda/internal/config/config.go | 6 +-- sda/internal/database/database.go | 4 +- sda/internal/database/db_functions.go | 12 +++--- sda/internal/helper/helper_test.go | 4 +- sda/internal/jsonadapter/jsonadapter_test.go | 2 +- sda/internal/storage/storage_test.go | 12 +++--- sda/internal/userauth/userauth_test.go | 2 +- 21 files changed, 59 insertions(+), 60 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index f6f0aaabd..0e080adaa 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -168,7 +168,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -229,7 +229,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -576,7 +576,7 @@ func (s *TestSuite) TearDownSuite() { s.GrpcListener.gs.GracefulStop() } if s.GrpcListener.Listener != nil { - s.GrpcListener.Listener.Close() + _ = s.GrpcListener.Listener.Close() } } func (s *TestSuite) SetupTest() { @@ -614,7 +614,7 @@ func (s *TestSuite) SetupTest() { req.SetBasicAuth("guest", "guest") res, err := client.Do(req) assert.NoError(s.T(), err, "failed to query broker") - res.Body.Close() + _ = res.Body.Close() } } @@ -867,8 +867,8 @@ func (s *TestSuite) TestGinLogLevel_Debug() { logOutput = buf.String() lines = strings.Split(strings.TrimSpace(logOutput), "\n") - fmt.Println("lines : ", lines) - fmt.Println("len(lines) : ", len(lines)) + _, _ = fmt.Println("lines : ", lines) + _, _ = fmt.Println("len(lines) : ", len(lines)) if len(lines) > 1 { assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") } else { @@ -944,8 +944,8 @@ func (s *TestSuite) TestGinLogLevel_Info() { logOutput = buf.String() lines = strings.Split(strings.TrimSpace(logOutput), "\n") - fmt.Println("lines : ", lines) - fmt.Println("len(lines) : ", len(lines)) + _, _ = fmt.Println("lines : ", lines) + _, _ = fmt.Println("len(lines) : ", len(lines)) if len(lines) > 1 { assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") } else { @@ -1121,7 +1121,7 @@ func (s *TestSuite) TestIngestFile_WithPayload() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -1278,7 +1278,7 @@ func (s *TestSuite) TestIngestFile_WithFileID() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -1403,7 +1403,7 @@ func (s *TestSuite) TestSetAccession_WithPayload() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -1516,7 +1516,7 @@ func (s *TestSuite) TestSetAccession_WithParams() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -1723,7 +1723,7 @@ func (s *TestSuite) TestCreateDataset() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") assert.NoError(s.T(), json.Unmarshal(body, &data), "failed to unmarshal response") assert.Equal(s.T(), 1, data.MessagesReady) @@ -1796,7 +1796,7 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { response := w.Result() body, err := io.ReadAll(response.Body) assert.NoError(s.T(), err) - response.Body.Close() + _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) assert.Contains(s.T(), string(body), "does not match pattern") @@ -1821,7 +1821,7 @@ func (s *TestSuite) TestCreateDataset_MissingAccessionIDs() { response := w.Result() body, err := io.ReadAll(response.Body) assert.NoError(s.T(), err) - response.Body.Close() + _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) assert.Contains(s.T(), string(body), "at least one accessionID is required") @@ -1845,7 +1845,7 @@ func (s *TestSuite) TestCreateDataset_WrongIDs() { response := w.Result() body, err := io.ReadAll(response.Body) assert.NoError(s.T(), err) - response.Body.Close() + _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) assert.Contains(s.T(), string(body), "accession ID not found: ") @@ -1904,7 +1904,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { response := w.Result() body, err := io.ReadAll(response.Body) assert.NoError(s.T(), err) - response.Body.Close() + _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) assert.Contains(s.T(), string(body), "accession ID owned by other user") @@ -1975,7 +1975,7 @@ func (s *TestSuite) TestReleaseDataset() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -2737,7 +2737,7 @@ func (s *TestSuite) TestReVerifyFile() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") @@ -2844,7 +2844,7 @@ func (s *TestSuite) TestReVerifyDataset() { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) assert.NoError(s.T(), err, "failed to unmarshal response") diff --git a/sda/cmd/auth/info_test.go b/sda/cmd/auth/info_test.go index ae46fc5ad..809c3e076 100644 --- a/sda/cmd/auth/info_test.go +++ b/sda/cmd/auth/info_test.go @@ -53,5 +53,5 @@ func (ts *InfoTests) TestReadPublicKeyFile() { } func (ts *InfoTests) TearDownTest() { - os.RemoveAll(ts.TempDir) + _ = os.RemoveAll(ts.TempDir) } diff --git a/sda/cmd/auth/jwt_test.go b/sda/cmd/auth/jwt_test.go index 055df14b3..316f92ac7 100644 --- a/sda/cmd/auth/jwt_test.go +++ b/sda/cmd/auth/jwt_test.go @@ -53,7 +53,7 @@ func (ts *JWTTests) SetupTest() { } func (ts *JWTTests) TearDownTest() { - os.RemoveAll(ts.TempDir) + _ = os.RemoveAll(ts.TempDir) } func (ts *JWTTests) TestGenerateJwtToken() { diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index f8f834245..4d5324dd6 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -202,7 +202,6 @@ func (app *Ingest) registerC4GHKey() error { } func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) string { - m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to update event log for file with id : %s", fileID) @@ -217,7 +216,6 @@ func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) st } func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) string { - status, err := app.DB.GetFileStatus(fileID) if err != nil && err.Error() != "sql: no rows in result set" { log.Errorf("failed to get status for file, fileID: %s, reason: (%s)", fileID, err.Error()) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index a5980352e..1ac59eacb 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -133,7 +133,7 @@ func TestMain(m *testing.M) { if err != nil || res.StatusCode != 200 { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { diff --git a/sda/cmd/notify/notify_test.go b/sda/cmd/notify/notify_test.go index 2bb29d1b0..44242491c 100644 --- a/sda/cmd/notify/notify_test.go +++ b/sda/cmd/notify/notify_test.go @@ -124,7 +124,7 @@ func TestSendEmail(t *testing.T) { }) if err := server.Start(); err != nil { - fmt.Println(err) + _, _ = fmt.Println(err) } hostAddress, portNumber := "127.0.0.1", server.PortNumber diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 99a39d1e3..55da6ebe1 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -86,7 +86,7 @@ func (ts *ReEncryptTests) SetupTest() { } func (ts *ReEncryptTests) TearDownTest() { - os.RemoveAll(ts.KeyPath) + _ = os.RemoveAll(ts.KeyPath) } func (ts *ReEncryptTests) TestReencryptHeader() { diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index b1734e0aa..d8ba19b1e 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -70,7 +70,7 @@ func main() { go func() { <-sigc // blocks here until it receives from sigc - fmt.Println("Interrupt signal received. Shutting down.") + _, _ = fmt.Println("Interrupt signal received. Shutting down.") defer app.MQ.Channel.Close() defer app.MQ.Connection.Close() defer app.DB.Close() diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index e4c8a4399..c2cd0cdb1 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -128,7 +128,7 @@ func TestMain(m *testing.M) { if err != nil || res.StatusCode != 200 { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -288,7 +288,6 @@ func (s *server) ReencryptHeader(ctx context.Context, req *re.ReencryptRequest) } func (ts *TestSuite) TestReEncryptHeader() { - for _, test := range []struct { corrID string expectedError error diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index e38be3ac1..03e12747f 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -140,7 +140,7 @@ func (s *ProxyTests) SetupTest() { ) _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(s.S3conf.Bucket)}) if err != nil { - fmt.Println(err.Error()) + _, _ = fmt.Println(err.Error()) } output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ @@ -176,11 +176,11 @@ func startFakeServer(port string) *FakeServer { log.Warnf("hello fake will return %s", f.resp) if f.resp != "" { log.Warnf("fake writes %s", f.resp) - fmt.Fprint(w, f.resp) + _, _ = fmt.Fprint(w, f.resp) } }) ts := httptest.NewUnstartedServer(foo) - ts.Listener.Close() + _ = ts.Listener.Close() ts.Listener = l ts.Start() diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index 5f2c7eae2..c0dcfae2b 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -78,7 +78,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -161,7 +161,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -218,7 +218,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { diff --git a/sda/cmd/syncapi/syncapi_test.go b/sda/cmd/syncapi/syncapi_test.go index da21dbab4..9eeeed92b 100644 --- a/sda/cmd/syncapi/syncapi_test.go +++ b/sda/cmd/syncapi/syncapi_test.go @@ -80,7 +80,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index 48a9c094e..c3807a7f1 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -141,7 +141,7 @@ func NewMQ(config MQConf) (*AMQPBroker, error) { } if e := channel.Confirm(false); e != nil { - fmt.Printf("channel could not be put into confirm mode: %s", e) + _, _ = fmt.Printf("channel could not be put into confirm mode: %s", e) return nil, fmt.Errorf("channel could not be put into confirm mode: %s", e) } diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 99c61420d..843e74b9d 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -86,7 +86,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -103,7 +103,7 @@ func TestMain(m *testing.M) { log.Panicf("Could not purge resource: %s", err) } - os.RemoveAll(certPath) + _ = os.RemoveAll(certPath) os.Exit(code) } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 840b7d521..58a08e2e9 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -1214,7 +1214,7 @@ func GetC4GHKey() (*[32]byte, error) { return nil, err } - keyFile.Close() + _ = keyFile.Close() return &key, nil } @@ -1236,7 +1236,7 @@ func GetC4GHprivateKeys() ([]*[32]byte, error) { } key, err := keys.ReadPrivateKey(keyFile, []byte(entry.Passphrase)) - keyFile.Close() + _ = keyFile.Close() if err != nil { return nil, fmt.Errorf("failed to read private key from %s: %v", entry.FilePath, err) } @@ -1260,7 +1260,7 @@ func GetC4GHPublicKey(keyPath string) (*[32]byte, error) { return nil, err } - keyFile.Close() + _ = keyFile.Close() return &key, nil } diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 71f6073b7..6e94c69ff 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -201,7 +201,7 @@ func (dbs *SDAdb) checkAndReconnectIfNeeded() { } func (dbs *SDAdb) Reconnect() { - dbs.DB.Close() + _ = dbs.DB.Close() dbs.DB, _ = sql.Open(dbs.Config.PgDataSource()) } @@ -213,6 +213,6 @@ func (dbs *SDAdb) Close() { err := dbs.DB.Ping() if err == nil { log.Info("Closing database connection") - dbs.DB.Close() + _ = dbs.DB.Close() } } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 84b731fec..728b03e62 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -31,7 +31,7 @@ func (dbs *SDAdb) RegisterFile(fileID *string, uploadPath, uploadUser string) (s query := "SELECT sda.register_file($1, $2, $3);" - var createdFileId string + var createdFileID string fileIDArg := sql.NullString{} if fileID != nil { @@ -39,9 +39,9 @@ func (dbs *SDAdb) RegisterFile(fileID *string, uploadPath, uploadUser string) (s fileIDArg.String = *fileID } - err := dbs.DB.QueryRow(query, fileIDArg, uploadPath, uploadUser).Scan(&createdFileId) + err := dbs.DB.QueryRow(query, fileIDArg, uploadPath, uploadUser).Scan(&createdFileID) - return createdFileId, err + return createdFileID, err } // GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID @@ -154,6 +154,7 @@ VALUES($1, $2, $3, $4, $5); if pqErr, ok := err.(*pq.Error); ok && pqErr.Code == "23503" { return sql.ErrNoRows } + return err } if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { @@ -843,6 +844,7 @@ ORDER BY f.id, fel.started_at DESC;` rows, err := db.Query(query, userID, pathPrefixArg, pathPrefixLen) if err != nil { log.Errorf("Error querying user files: %v", err) + return nil, err } defer rows.Close() @@ -1107,7 +1109,7 @@ func (dbs *SDAdb) ListDatasets() ([]*DatasetInfo, error) { datasets = append(datasets, &di) } - rows.Close() + _ = rows.Close() return datasets, nil } @@ -1149,7 +1151,7 @@ func (dbs *SDAdb) ListUserDatasets(submissionUser string) ([]DatasetInfo, error) datasets = append(datasets, di) } - rows.Close() + _ = rows.Close() return datasets, nil } diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 01c9227ce..06b1dedb5 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -68,7 +68,7 @@ func (ts *HelperTest) TestCreateRSAToken() { assert.NoError(ts.T(), err) assert.NoError(ts.T(), set.AddKey(key)) - fmt.Println(tok) + _, _ = fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) assert.NoError(ts.T(), err) @@ -120,7 +120,7 @@ func (ts *HelperTest) TestCreateHSToken() { assert.NoError(ts.T(), set.AddKey(jwtKey)) - fmt.Println(tok) + _, _ = fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) assert.NoError(ts.T(), err) diff --git a/sda/internal/jsonadapter/jsonadapter_test.go b/sda/internal/jsonadapter/jsonadapter_test.go index 406a02c52..d501c932f 100644 --- a/sda/internal/jsonadapter/jsonadapter_test.go +++ b/sda/internal/jsonadapter/jsonadapter_test.go @@ -52,7 +52,7 @@ func (ts *AdapterTestSuite) SetupSuite() { } func (ts *AdapterTestSuite) TearDownSuite() { - os.RemoveAll(ts.File.Name()) + _ = os.RemoveAll(ts.File.Name()) } func (ts *AdapterTestSuite) TestAdapter_empty() { diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index a97819b19..67b626dc2 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -111,7 +111,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { @@ -131,7 +131,7 @@ func TestMain(m *testing.M) { log.Panicf("Could not purge resource: %s", err) } - os.RemoveAll(sshPath) + _ = os.RemoveAll(sshPath) os.Exit(code) } @@ -234,7 +234,7 @@ func (ts *StorageTestSuite) TestPosixBackend() { written, err := writer.Write(writeData) assert.NoError(ts.T(), err, "Failure when writing to posix writer") assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - writer.Close() + _ = writer.Close() reader, err := backend.NewFileReader("testFile") assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") @@ -315,7 +315,7 @@ func (ts *StorageTestSuite) TestS3Backend() { written, err := writer.Write(writeData) assert.Nil(ts.T(), err, "Failure when writing to s3 writer") assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - writer.Close() + _ = writer.Close() // sleep to allow the write to complete, otherwise the next step will fail due to timing issues. time.Sleep(1 * time.Second) @@ -341,7 +341,7 @@ func (ts *StorageTestSuite) TestS3Backend() { written, err = writer.Write(writeData) assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") assert.Nil(ts.T(), err, "Failure when writing to s3 writer") - writer.Close() + _ = writer.Close() size, err = s3back.GetFileSize("s3Creatable", true) assert.Nil(ts.T(), err, "s3 GetFileSize with expected delay failed when it should work") assert.NotNil(ts.T(), size, "Got a nil size for s3") @@ -390,7 +390,7 @@ func (ts *StorageTestSuite) TestSftpBackend() { written, err := writer.Write(writeData) assert.Nil(ts.T(), err, "Failure when writing to sftp writer") assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - writer.Close() + _ = writer.Close() reader, err := sftpBack.NewFileReader(sftpCreatable) assert.Nil(ts.T(), err, "sftp NewFileReader failed when it should work") diff --git a/sda/internal/userauth/userauth_test.go b/sda/internal/userauth/userauth_test.go index 1a984172c..4de8aa0c0 100644 --- a/sda/internal/userauth/userauth_test.go +++ b/sda/internal/userauth/userauth_test.go @@ -94,7 +94,7 @@ func TestMain(m *testing.M) { if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() return nil }); err != nil { From 1fb4ba5289a90f890af32cb80dfc89c620386710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 16:24:45 +0100 Subject: [PATCH 179/184] feat(postgres): fix migration scripts version and comments, ensure they are not applied again if db inited with those changes, add removal of unused pg funcs, update sanity check to allow migration files to contain descriptive names after version, fix postgres integration tests --- .../tests/postgres/10_sanity_check.sh | 2 +- .../tests/postgres/20_inbox_queries.sh | 6 ++-- .../tests/postgres/30_ingest_queries.sh | 24 +++++++++---- .../tests/postgres/40_verify_queries.sh | 36 ++++++++++++++----- Makefile | 5 +++ postgresql/initdb.d/01_main.sql | 7 +++- postgresql/initdb.d/02_functions.sql | 30 ---------------- ..._add_index_on_files_and_file_event_log.sql | 2 +- ...precate_files_event_log_correlation_id.sql | 4 +-- ...unctions_set_verified_and_set_archived.sql | 26 ++++++++++++++ 10 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 postgresql/migratedb.d/21_drop_unused_functions_set_verified_and_set_archived.sql diff --git a/.github/integration/tests/postgres/10_sanity_check.sh b/.github/integration/tests/postgres/10_sanity_check.sh index af7b270ba..076815266 100644 --- a/.github/integration/tests/postgres/10_sanity_check.sh +++ b/.github/integration/tests/postgres/10_sanity_check.sh @@ -22,7 +22,7 @@ if [ "$status" -eq 0 ]; then fi ## verify that migrations worked -migratedb=$(find /migratedb.d/ -name "*.sql" -printf '%f\n' | sort -n | tail -1 | cut -d '.' -f1) +migratedb=$(find /migratedb.d/ -name "*.sql" -printf '%f\n' | sort -n | tail -1 | cut -d '.' -f1 | cut -d '_' -f1) version=$(psql -U postgres -h migrate -d sda -At -c "select max(version) from sda.dbschema_version;") if [ "$version" -ne "$migratedb" ]; then echo "Migration scripts failed" diff --git a/.github/integration/tests/postgres/20_inbox_queries.sh b/.github/integration/tests/postgres/20_inbox_queries.sh index f2e8d51be..43f39c027 100644 --- a/.github/integration/tests/postgres/20_inbox_queries.sh +++ b/.github/integration/tests/postgres/20_inbox_queries.sh @@ -1,16 +1,16 @@ #!/bin/sh set -eou pipefail - +fileID="33d29907-c565-4a90-98b4-e31b992ab376" export PGPASSWORD=inbox for host in migrate postgres; do - fileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'test-user');") + fileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('$fileID', 'inbox/test-file.c4gh', 'test-user');") if [ -z "$fileID" ]; then echo "register_file failed" exit 1 fi - newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'other-user');") + newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file(null, 'inbox/test-file.c4gh', 'other-user');") if [ -z "$newFileID" ]; then echo "register_file failed" exit 1 diff --git a/.github/integration/tests/postgres/30_ingest_queries.sh b/.github/integration/tests/postgres/30_ingest_queries.sh index 283ae557b..8801bf31d 100644 --- a/.github/integration/tests/postgres/30_ingest_queries.sh +++ b/.github/integration/tests/postgres/30_ingest_queries.sh @@ -3,17 +3,17 @@ set -eou pipefail export PGPASSWORD=ingest user="test-user" -corrID="33d29907-c565-4a90-98b4-e31b992ab376" +fileID="33d29907-c565-4a90-98b4-e31b992ab376" for host in migrate postgres; do ## insert file - fileID=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', '$user');") + fileID=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.register_file('$fileID', 'inbox/test-file.c4gh', '$user');") if [ -z "$fileID" ]; then echo "register_file failed" exit 1 fi - resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$fileID', 'submitted', '$corrID', '$user', '{}');") + resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, user_id, message) VALUES('$fileID', 'submitted', '$user', '{}');") if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then echo "insert file failed" exit 1 @@ -30,11 +30,23 @@ for host in migrate postgres; do archive_path=d853c51b-6aed-4243-b427-177f5e588857 size="2035150" checksum="f03775a50feea74c579d459fdbeb27adafd543b87f6692703543a6ebe7daa1ff" - resp=$(psql -U ingest -h "$host" -d sda -At -c "SELECT sda.set_archived('$fileID', '$corrID', '$archive_path', '$size', '$checksum', 'SHA256');") - if [ "$resp" != "" ]; then - echo "mark file archived failed" + + resp=$(psql -U ingest -h "$host" -d sda -At -c "UPDATE sda.files SET archive_file_path = '$archive_path', archive_file_size = '$size' WHERE id = '$fileID';") + if [ "$resp" != "UPDATE 1" ]; then + echo "update of files.archive_file_path, archive_file_size failed: $resp" + exit 1 + fi + resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$checksum', upper('SHA256')::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source);") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert of archived checksum failed: $resp" exit 1 fi + resp=$(psql -U ingest -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$fileID', 'archived');") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert of file_event_log failed: $resp" + exit 1 + fi + done echo "30_ingest_queries completed successfully" diff --git a/.github/integration/tests/postgres/40_verify_queries.sh b/.github/integration/tests/postgres/40_verify_queries.sh index 4b3653aaa..99e87b53c 100644 --- a/.github/integration/tests/postgres/40_verify_queries.sh +++ b/.github/integration/tests/postgres/40_verify_queries.sh @@ -2,15 +2,14 @@ set -eou pipefail export PGPASSWORD=verify -corrID="33d29907-c565-4a90-98b4-e31b992ab376" +fileID="33d29907-c565-4a90-98b4-e31b992ab376" for host in migrate postgres; do - fileID=$(psql -U verify -h "$host" -d sda -At -c "SELECT DISTINCT file_id from sda.file_event_log WHERE correlation_id = '$corrID';") ## get file status - status=$(psql -U verify -h "$host" -d sda -At -c "SELECT event from sda.file_event_log WHERE correlation_id = '$corrID' ORDER BY id DESC LIMIT 1;") + status=$(psql -U verify -h "$host" -d sda -At -c "SELECT event from sda.file_event_log WHERE file_id = '$fileID' ORDER BY id DESC LIMIT 1;") if [ "$status" = "" ]; then - echo "get file status failed" + echo "get file status failed: $resp" exit 1 fi @@ -18,7 +17,7 @@ for host in migrate postgres; do header="637279707434676801000000010000006c00000000000000" dbheader=$(psql -U verify -h "$host" -d sda -At -c "SELECT header from sda.files WHERE id = '$fileID';") if [ "$dbheader" != "$header" ]; then - echo "wrong header received" + echo "wrong header received: $resp" exit 1 fi @@ -26,11 +25,32 @@ for host in migrate postgres; do archive_checksum="64e56b0d245b819c116b5f1ad296632019490b57eeaebb419a5317e24a153852" decrypted_size="2034254" decrypted_checksum="febee6829a05772eea93c647e38bf5cc5bf33d1bcd0ea7d7bdd03225d84d2553" - resp=$(psql -U verify -h "$host" -d sda -At -c "SELECT sda.set_verified('$fileID', '$corrID', '$archive_checksum', 'SHA256', '$decrypted_size', '$decrypted_checksum', 'SHA256')") - if [ "$resp" != "" ]; then - echo "set_verified failed" + + resp=$(psql -U verify -h "$host" -d sda -At -c "UPDATE sda.files SET decrypted_file_size = '$decrypted_size' WHERE id = '$fileID';") + if [ "$resp" != "UPDATE 1" ]; then + echo "update of files.decrypted_file_size failed: $resp" + exit 1 + fi + + resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$archive_checksum', upper('SHA256')::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source);") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert of archived checksum failed: $resp" + exit 1 + fi + + resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.checksums(file_id, checksum, type, source) VALUES('$fileID', '$decrypted_checksum', upper('SHA256')::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source);") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert of decrypted checksum failed: $resp" + exit 1 + fi + + resp=$(psql -U verify -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event) VALUES('$fileID', 'verified');") + if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then + echo "insert of file_event_log failed: $resp" exit 1 fi + + done echo "40_verify_queries completed successfully" diff --git a/Makefile b/Makefile index ca0f91444..c2c1f7f32 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,11 @@ sda-sync-down: integrationtest-postgres: build-postgresql @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml run tests @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml down -v --remove-orphans +integrationtest-postgres-run: build-postgresql + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml run tests +integrationtest-postgres-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/postgres.yml down -v --remove-orphans + integrationtest-rabbitmq: build-rabbitmq build-sda @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/rabbitmq-federation.yml down -v --remove-orphans diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index e0b710577..97cfdf6bb 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -30,7 +30,12 @@ VALUES (0, now(), 'Created with version'), (13, now(), 'Create API user'), (14, now(), 'Create Auth user'), (15, now(), 'Give API user insert priviledge in logs table'), - (16, now(), 'Give ingest user select priviledge in encryption_keys table'); + (16, now(), 'Give ingest user select priviledge in encryption_keys table'), + (17, now(), 'Add submission user to constraint'), + (18, now(), 'Create rotatekey role and grant it priviledges to sda tables'), + (19, now(), 'Create new indexes on files and file_event_log tables'), + (20, now(), 'Deprecate file_event_log.correlation_id column and migrate data where file_id != correlation_id'), + (21, now(), 'Drop functions set_verified, and set_archived'); -- Datasets are used to group files, and permissions are set on the dataset -- level diff --git a/postgresql/initdb.d/02_functions.sql b/postgresql/initdb.d/02_functions.sql index b80231793..5b33670b1 100644 --- a/postgresql/initdb.d/02_functions.sql +++ b/postgresql/initdb.d/02_functions.sql @@ -42,33 +42,3 @@ VALUES (file_uuid, 'registered', submission_user); RETURN file_uuid; END; $register_file$ LANGUAGE plpgsql; - - -CREATE FUNCTION set_archived(file_uuid UUID, corr_id UUID, file_path TEXT, file_size BIGINT, inbox_checksum_value TEXT, inbox_checksum_type TEXT) -RETURNS void AS $set_archived$ -BEGIN - UPDATE sda.files SET archive_file_path = file_path, archive_file_size = file_size WHERE id = file_uuid; - - INSERT INTO sda.checksums(file_id, checksum, type, source) - VALUES(file_uuid, inbox_checksum_value, upper(inbox_checksum_type)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source); - - INSERT INTO sda.file_event_log(file_id, event, correlation_id) VALUES(file_uuid, 'archived', corr_id); -END; - -$set_archived$ LANGUAGE plpgsql; - -CREATE FUNCTION set_verified(file_uuid UUID, corr_id UUID, archive_checksum TEXT, archive_checksum_type TEXT, decrypted_size BIGINT, decrypted_checksum TEXT, decrypted_checksum_type TEXT) -RETURNS void AS $set_verified$ -BEGIN - UPDATE sda.files SET decrypted_file_size = decrypted_size WHERE id = file_uuid; - - INSERT INTO sda.checksums(file_id, checksum, type, source) - VALUES(file_uuid, archive_checksum, upper(archive_checksum_type)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source); - - INSERT INTO sda.checksums(file_id, checksum, type, source) - VALUES(file_uuid, decrypted_checksum, upper(decrypted_checksum_type)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source); - - INSERT INTO sda.file_event_log(file_id, event, correlation_id) VALUES(file_uuid, 'verified', corr_id); -END; - -$set_verified$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql b/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql index d1c378576..c559b3980 100644 --- a/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql +++ b/postgresql/migratedb.d/19_add_index_on_files_and_file_event_log.sql @@ -5,7 +5,7 @@ DECLARE -- The version we know how to do migration from, at the end of a successful migration -- we will no longer be at this version. sourcever INTEGER := 18; - changes VARCHAR := 'Create new indexes on files and file_event_log tables, and new generated column submission_file_root_dir on files table'; + changes VARCHAR := 'Create new indexes on files and file_event_log tables'; BEGIN IF (select max(version) from sda.dbschema_version) = sourcever then RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; diff --git a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql index 21e77bc7f..4b7e5e4ae 100644 --- a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql +++ b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql @@ -4,8 +4,8 @@ $$ DECLARE -- The version we know how to do migration from, at the end of a successful migration -- we will no longer be at this version. - sourcever INTEGER := 17; - changes VARCHAR := 'Create rotatekey role and grant it priviledges to sda tables'; + sourcever INTEGER := 19; + changes VARCHAR := 'Deprecate file_event_log.correlation_id column and migrate data where file_id != correlation_id'; BEGIN IF (select max(version) from sda.dbschema_version) = sourcever then RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; diff --git a/postgresql/migratedb.d/21_drop_unused_functions_set_verified_and_set_archived.sql b/postgresql/migratedb.d/21_drop_unused_functions_set_verified_and_set_archived.sql new file mode 100644 index 000000000..bde25978c --- /dev/null +++ b/postgresql/migratedb.d/21_drop_unused_functions_set_verified_and_set_archived.sql @@ -0,0 +1,26 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 20; + changes VARCHAR := 'Drop functions set_verified, and set_archived'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + + -- Drop set_verified func, as not in use + DROP FUNCTION IF EXISTS sda.set_verified; + + -- Drop set_archived func, as not in use + DROP FUNCTION IF EXISTS sda.set_archived; + +ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ From e69e4550270a94ab740722f067e1e0c0f0f3f21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 1 Dec 2025 16:38:11 +0100 Subject: [PATCH 180/184] feat(rotatekey): fix rotatekey test, and update reEncryptHeader func to not take correlation id just file id and publish message with fileid instead --- sda/cmd/rotatekey/rotatekey.go | 6 +++--- sda/cmd/rotatekey/rotatekey_test.go | 9 ++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index d8ba19b1e..bf0302556 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -161,7 +161,7 @@ func main() { // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - ackNack, msg, err := app.reEncryptHeader(delivered.CorrelationId, message.FileID) + ackNack, msg, err := app.reEncryptHeader(message.FileID) switch ackNack { case "ack": @@ -197,7 +197,7 @@ func main() { <-forever } -func (app *RotateKey) reEncryptHeader(correlationID, fileID string) (ackNack, msg string, err error) { +func (app *RotateKey) reEncryptHeader(fileID string) (ackNack, msg string, err error) { // Get current keyhash for the file, send to error queue if this fails oldKeyHash, err := app.DB.GetKeyHash(fileID) if err != nil { @@ -270,7 +270,7 @@ func (app *RotateKey) reEncryptHeader(correlationID, fileID string) (ackNack, ms return "ackSendToError", msg, err } - if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "archived", reVerifyMsg); err != nil { msg := "failed to publish message" log.Errorf("%s, reason: %v", msg, err) diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index c2cd0cdb1..4de3af6bc 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -288,8 +288,8 @@ func (s *server) ReencryptHeader(ctx context.Context, req *re.ReencryptRequest) } func (ts *TestSuite) TestReEncryptHeader() { + newFileID := uuid.NewString() for _, test := range []struct { - corrID string expectedError error expectedMgs string expectedRes string @@ -306,14 +306,13 @@ func (ts *TestSuite) TestReEncryptHeader() { { testName: "un-ingested file", expectedError: errors.New("sql: no rows in result set"), - expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", ts.fileID), + expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", newFileID), expectedRes: "ackSendToError", - corrID: uuid.New().String(), - fileID: ts.fileID, + fileID: newFileID, }, } { ts.T().Run(test.testName, func(t *testing.T) { - res, msg, err := ts.app.reEncryptHeader(test.corrID, test.fileID) + res, msg, err := ts.app.reEncryptHeader(test.fileID) assert.Equal(t, res, test.expectedRes) assert.Equal(t, msg, test.expectedMgs) assert.Equal(t, err, test.expectedError) From 7ecbf3f5291aecfbaa225528df1d22aad5aad996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 2 Dec 2025 10:18:34 +0100 Subject: [PATCH 181/184] feat(api): revert change to getFileIDByUserPathAndStatus to check for null stable_id, introduce new database func CheckStableIDOwnedByUser to be used when creating dataset to check if a file belongs to user by stableID(aka accessionID) --- sda/cmd/api/api.go | 32 +++++--------------- sda/cmd/api/api_test.go | 4 +-- sda/internal/database/db_functions.go | 42 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 85384c9c4..de5ceb8b6 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -699,35 +699,19 @@ func createDataset(c *gin.Context) { return } + // Check that the files the accession ids are linked to belong to the user of the dataset for _, stableID := range dataset.AccessionIDs { - inboxPath, err := Conf.API.DB.GetInboxPath(stableID) + belongsToUser, err := Conf.API.DB.CheckStableIDOwnedByUser(stableID, dataset.User) if err != nil { - switch { - case err.Error() == "sql: no rows in result set": - log.Errorln(err.Error()) - c.AbortWithStatusJSON(http.StatusBadRequest, fmt.Sprintf("accession ID not found: %s", stableID)) - - return - default: - log.Errorln(err.Error()) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + log.Errorln(err.Error()) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - return - } + return } - if _, err := Conf.API.DB.GetFileIDByUserPathAndStatus(dataset.User, inboxPath, "ready"); err != nil { - switch { - case err.Error() == "sql: no rows in result set": - log.Errorln(err.Error()) - c.AbortWithStatusJSON(http.StatusBadRequest, "accession ID owned by other user") - - return - default: - log.Errorln(err.Error()) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + if !belongsToUser { + c.AbortWithStatusJSON(http.StatusBadRequest, fmt.Sprintf("accession ID: %s not found or owned by other user", stableID)) - return - } + return } } diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 0e080adaa..e7d3e9e29 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1848,7 +1848,7 @@ func (s *TestSuite) TestCreateDataset_WrongIDs() { _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(s.T(), string(body), "accession ID not found: ") + assert.Contains(s.T(), string(body), "accession ID: API:accession-id-11 not found or owned by other user") } func (s *TestSuite) TestCreateDataset_WrongUser() { @@ -1907,7 +1907,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { _ = response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(s.T(), string(body), "accession ID owned by other user") + assert.Contains(s.T(), string(body), "accession ID: API:accession-id-11 not found or owned by other user") } func (s *TestSuite) TestReleaseDataset() { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 728b03e62..99f754efc 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -110,6 +110,7 @@ FROM ( LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id WHERE f.submission_user = $1 AND f.submission_file_path = $2 + AND f.stable_id IS NULL ORDER BY f.id, fel.started_at DESC LIMIT 1 ) AS id_and_event WHERE id_and_event.event = $3;` @@ -123,6 +124,47 @@ WHERE id_and_event.event = $3;` return fileID, nil } +// CheckStableIDOwnedByUser checks if the file a stableID links to belongs to the user +// Returns true if a file is found by the stableID and user, false if not found +func (dbs *SDAdb) CheckStableIDOwnedByUser(stableID, user string) (bool, error) { + var ( + err error + found bool + ) + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + found, err = dbs.checkStableIDOwnedByUser(stableID, user) + if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return found, err +} + +func (dbs *SDAdb) checkStableIDOwnedByUser(stableID, user string) (bool, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const checkFileFound = ` +SELECT true +FROM sda.files +WHERE stable_id = $1 +AND submission_user = $2` + + var found bool + if err := db.QueryRow(checkFileFound, stableID, user).Scan(&found); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + + return false, err + } + + return true, nil +} + // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, user, details, message string) error { From b08814c4794982bf8ce2ef5580842ef59f843c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 3 Dec 2025 12:14:24 +0100 Subject: [PATCH 182/184] feat: update some comments, and logs to say fileID instead of correlationID where relevant, and update message handling funcs to use fileID variable instead of delivered.CorrelationId to reduce risk of confusion --- sda/cmd/api/api.go | 6 ++---- sda/cmd/finalize/finalize.go | 18 ++++++++--------- sda/cmd/ingest/ingest.go | 8 ++++---- sda/cmd/intercept/intercept.go | 2 +- sda/cmd/mapper/mapper.go | 2 +- sda/cmd/orchestrate/orchestrate.go | 2 +- sda/cmd/rotatekey/rotatekey.go | 2 +- sda/cmd/sync/sync.go | 2 +- sda/cmd/verify/verify.go | 32 +++++++++++++++--------------- 9 files changed, 36 insertions(+), 38 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index de5ceb8b6..df629efa5 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -323,7 +323,7 @@ This endpoint supports two input modes: 1. By file ID (via the "fileid" query parameter): Looks up the user and file path from the database. 2. By JSON payload: Expects a JSON body with user and file path. The function constructs an ingest message, validates it -and sends it to the broker with the appropriate correlation ID. +and sends it to the broker with the appropriate file ID. */ func ingestFile(c *gin.Context) { var ( @@ -361,7 +361,6 @@ func ingestFile(c *gin.Context) { return } - // Find the correlation id of the file fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(ingest.User, ingest.FilePath, "uploaded") if err != nil { if fileID == "" { @@ -575,7 +574,7 @@ func downloadFile(c *gin.Context) { setAccession handles requests to assign an accession ID to a file. This endpoint supports two input modes: 1. By query parameters ("fileid" and "accessionid"): Retrieves user, file path, and decrypted checksum from the database using the file ID. -2. By JSON payload: Expects a JSON body with user and file path, then looks up the correlation ID and decrypted checksum. +2. By JSON payload: Expects a JSON body with user and file path, then looks up the file ID and decrypted checksum. If both query parameters and a JSON payload are provided, the request is rejected with a 400 Bad Request. The function constructs an accession message, validates it and sends it to the message broker. */ @@ -631,7 +630,6 @@ func setAccession(c *gin.Context) { return } - // Find the correlation id fileID, err = Conf.API.DB.GetFileIDByUserPathAndStatus(accession.User, accession.FilePath, "verified") if err != nil { if fileID == "" { diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 486c9162a..59bbabdc8 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -73,7 +73,7 @@ func main() { log.Fatal(err) } for delivered := range messages { - log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) @@ -84,12 +84,13 @@ func main() { continue } + fileID := delivered.CorrelationId // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(delivered.CorrelationId) + status, err := db.GetFileStatus(fileID) if err != nil { - log.Errorf("failed to get file status, correlation-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to get file status, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -99,7 +100,7 @@ func main() { switch status { case "disabled": - log.Infof("file with correlation-id: %s is disabled, aborting work", delivered.CorrelationId) + log.Infof("file with file-id: %s is disabled, aborting work", fileID) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -108,14 +109,14 @@ func main() { case "verified", "enabled": case "ready": - log.Infof("File with correlation-id: %s is already marked as ready.", delivered.CorrelationId) + log.Infof("File with file-id: %s is already marked as ready.", fileID) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking message, reason: %v", err) } continue default: - log.Warnf("file with correlation-id: %s is not verified yet, aborting work", delivered.CorrelationId) + log.Warnf("file with file-id: %s is not verified yet, aborting work", fileID) if err := delivered.Nack(false, true); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -123,7 +124,6 @@ func main() { continue } - fileID := delivered.CorrelationId c := schema.IngestionCompletion{ User: message.User, FilePath: message.FilePath, @@ -160,7 +160,7 @@ func main() { body, _ := json.Marshal(fileError) // Send the message to an error queue so it can be analyzed. - if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { + if e := mq.SendMessage(fileID, conf.Broker.Exchange, "error", body); e != nil { log.Errorf("failed to publish message, reason: %v", err) } @@ -203,7 +203,7 @@ func main() { continue } - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { + if err := mq.SendMessage(fileID, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { log.Errorf("failed to publish message, reason: %v", err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4d5324dd6..2c4a70271 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -123,7 +123,7 @@ func main() { } for delivered := range messages { - log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) message := schema.IngestionTrigger{} err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { @@ -148,7 +148,7 @@ func main() { // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) - log.Infof("Received work (corr-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) + log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) ackNack := "" switch message.Type { @@ -309,7 +309,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st } case "": // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox - log.Infof("registering file, correlation-id: %s", fileID) + log.Infof("registering file, file-id: %s", fileID) fileID, err = app.DB.RegisterFile(&fileID, message.FilePath, message.User) if err != nil { log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, err.Error()) @@ -319,7 +319,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st case "uploaded": default: - log.Warnf("unsupported file status: %s, correlation-id: %s", status, fileID) + log.Warnf("unsupported file status: %s, file-id: %s", status, fileID) return "reject" } diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index 64fd0e4dc..137f63bfe 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -90,7 +90,7 @@ func main() { continue } - log.Infof("Routing message (corr-id: %s, routingkey: %s)", delivered.CorrelationId, routingKey) + log.Infof("Routing message (correlation-id: %s, routingkey: %s)", delivered.CorrelationId, routingKey) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, routingKey, delivered.Body); err != nil { log.Errorf("failed to publish message, reason: (%v)", err) } diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 189264de3..c4a675aac 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -104,7 +104,7 @@ func main() { } for _, aID := range mappings.AccessionIDs { - log.Debugf("Mapped file to dataset (corr-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) + log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) fileInfo, err := db.GetFileInfoFromAccessionID(aID) if err != nil { log.Errorf("failed to get file info for file with stable ID: %s", aID) diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index 457ac838a..53b8d2fdb 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -381,7 +381,7 @@ func validateMsg(delivered *amqp091.Delivery, mq *broker.AMQPBroker, routingKey return err } - log.Debugf("Routing message (corr-id: %s, routingkey: %s, message: %s)", delivered.CorrelationId, routingKey, publishMsg) + log.Debugf("Routing message (correlation-id: %s, routingkey: %s, message: %s)", delivered.CorrelationId, routingKey, publishMsg) if err := mq.SendMessage(delivered.CorrelationId, mq.Conf.Exchange, routingKey, publishMsg); err != nil { // TODO fix resend mechanism diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go index bf0302556..989c92e4c 100644 --- a/sda/cmd/rotatekey/rotatekey.go +++ b/sda/cmd/rotatekey/rotatekey.go @@ -125,7 +125,7 @@ func main() { panic(err) } for delivered := range messages { - log.Debugf("Received a message (corr-id: %s, message: %s)", + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 2c7ac2977..67c174136 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -85,7 +85,7 @@ func main() { log.Fatal(err) } for delivered := range messages { - log.Debugf("Received a message (corr-id: %s, message: %s)", + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index c26d22bf4..fc0a008bd 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -71,7 +71,7 @@ func main() { err) } for delivered := range messages { - log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) @@ -97,14 +97,14 @@ func main() { _ = json.Unmarshal(delivered.Body, &message) log.Infof( - "Received work (corr-id: %s, filepath: %s, user: %s)", - delivered.CorrelationId, message.FilePath, message.User, + "Received work (message.correlation-id: %s, file-id: %s, filepath: %s, user: %s)", + delivered.CorrelationId, message.FileID, message.FilePath, message.User, ) // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(delivered.CorrelationId) + status, err := db.GetFileStatus(message.FileID) if err != nil { - log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -113,7 +113,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } @@ -124,7 +124,7 @@ func main() { continue } if status == "disabled" { - log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } @@ -148,7 +148,7 @@ func main() { body, _ := json.Marshal(infoErrorMessage) // Send the message to an error queue so it can be analyzed. - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } @@ -177,7 +177,7 @@ func main() { OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } @@ -196,7 +196,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } @@ -243,7 +243,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) } @@ -323,9 +323,9 @@ func main() { // Logging is in ValidateJSON so just restart on new message continue } - status, err := db.GetFileStatus(delivered.CorrelationId) + status, err := db.GetFileStatus(message.FileID) if err != nil { - log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -334,7 +334,7 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } @@ -346,7 +346,7 @@ func main() { } if status == "disabled" { - log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } @@ -387,7 +387,7 @@ func main() { } // Send message to verified queue - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { // TODO fix resend mechanism log.Errorf("failed to publish message, reason: (%s)", err.Error()) From 91ff1bdbdee08a79c3127fed030a2b8fce450888 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 3 Dec 2025 13:29:32 +0100 Subject: [PATCH 183/184] feat(postgres): fix typo in changes messages for previous migrations verisons: 15, 16, and 18 priviledge -> privilege --- postgresql/initdb.d/01_main.sql | 6 +++--- postgresql/migratedb.d/15.sql | 2 +- postgresql/migratedb.d/16.sql | 2 +- postgresql/migratedb.d/18.sql | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index 97cfdf6bb..29ec12198 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -29,10 +29,10 @@ VALUES (0, now(), 'Created with version'), (12, now(), 'Add key hash'), (13, now(), 'Create API user'), (14, now(), 'Create Auth user'), - (15, now(), 'Give API user insert priviledge in logs table'), - (16, now(), 'Give ingest user select priviledge in encryption_keys table'), + (15, now(), 'Give API user insert privilege in logs table'), + (16, now(), 'Give ingest user select privilege in encryption_keys table'), (17, now(), 'Add submission user to constraint'), - (18, now(), 'Create rotatekey role and grant it priviledges to sda tables'), + (18, now(), 'Create rotatekey role and grant it privileges to sda tables'), (19, now(), 'Create new indexes on files and file_event_log tables'), (20, now(), 'Deprecate file_event_log.correlation_id column and migrate data where file_id != correlation_id'), (21, now(), 'Drop functions set_verified, and set_archived'); diff --git a/postgresql/migratedb.d/15.sql b/postgresql/migratedb.d/15.sql index 2849382c4..d58135fbc 100644 --- a/postgresql/migratedb.d/15.sql +++ b/postgresql/migratedb.d/15.sql @@ -5,7 +5,7 @@ DECLARE -- The version we know how to do migration from, at the end of a successful migration -- we will no longer be at this version. sourcever INTEGER := 14; - changes VARCHAR := 'Give API user insert priviledge in logs table'; + changes VARCHAR := 'Give API user insert privilege in logs table'; BEGIN IF (select max(version) from sda.dbschema_version) = sourcever then RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; diff --git a/postgresql/migratedb.d/16.sql b/postgresql/migratedb.d/16.sql index 138e4fb9c..1783a051a 100644 --- a/postgresql/migratedb.d/16.sql +++ b/postgresql/migratedb.d/16.sql @@ -5,7 +5,7 @@ DECLARE -- The version we know how to do migration from, at the end of a successful migration -- we will no longer be at this version. sourcever INTEGER := 15; - changes VARCHAR := 'Give ingest user select priviledge in encryption_keys table'; + changes VARCHAR := 'Give ingest user select privilege in encryption_keys table'; BEGIN IF (select max(version) from sda.dbschema_version) = sourcever then RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; diff --git a/postgresql/migratedb.d/18.sql b/postgresql/migratedb.d/18.sql index 92effc43f..036128875 100644 --- a/postgresql/migratedb.d/18.sql +++ b/postgresql/migratedb.d/18.sql @@ -5,7 +5,7 @@ DECLARE -- The version we know how to do migration from, at the end of a successful migration -- we will no longer be at this version. sourcever INTEGER := 17; - changes VARCHAR := 'Create rotatekey role and grant it priviledges to sda tables'; + changes VARCHAR := 'Create rotatekey role and grant it privileges to sda tables'; BEGIN IF (select max(version) from sda.dbschema_version) = sourcever then RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; From 511e49e4f58f32933c1b4a6da8c23365ca99d9db Mon Sep 17 00:00:00 2001 From: Krzysztof Kochel Date: Thu, 4 Dec 2025 23:28:57 +0100 Subject: [PATCH 184/184] Fix after migration --- .github/integration/sda-s3-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index bc852002d..266695e00 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -254,7 +254,7 @@ services: - shared:/shared rotatekey: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [sda-rotatekey] container_name: rotatekey depends_on: