repository: do not crash on missing segment files/dirs, fixes #3225

src/borg/repository.py

@@ -1470,11 +1470,13 @@ def get_write_fd(self, no_new=False, want_new=False, raise_full=False):
                 raise self.SegmentFull
             self.close_segment()
         if not self._write_fd:
-            if self.segment % self.segments_per_dir == 0:
-                dirname = os.path.join(self.path, 'data', str(self.segment // self.segments_per_dir))
-                if not os.path.exists(dirname):
-                    os.mkdir(dirname)
-                    sync_dir(os.path.join(self.path, 'data'))
+            # usually the segment dir already exists, but it does not when we start a new one and
+            # it also might be missing in a damaged repository (e.g. after a filesystem issue),
+            # so always check for it rather than only for the first segment of a dir, see #3225.
+            dirname = os.path.join(self.path, 'data', str(self.segment // self.segments_per_dir))
+            if not os.path.exists(dirname):
+                os.makedirs(dirname, exist_ok=True)
+                sync_dir(os.path.join(self.path, 'data'))
             self._write_fd = SyncFile(self.segment_filename(self.segment), binary=True)
             self._write_fd.write(MAGIC)
             self.offset = MAGIC_LEN
@@ -1492,7 +1494,16 @@ def get_fd(self, segment):
         now = time.monotonic()
 
         def open_fd():
-            fd = open(self.segment_filename(segment), 'rb')
+            filename = self.segment_filename(segment)
+            try:
+                fd = open(filename, 'rb')
+            except FileNotFoundError:
+                # the repository is damaged: something (or somebody) removed a segment file that
+                # the repository index still refers to. give a clear error instead of a
+                # FileNotFoundError traceback, see #3225.
+                raise IntegrityError(f'Segment file {filename} is missing. '
+                                     'Run borg check (full check, not --archives-only) '
+                                     'and then borg check --repair to fix the repository.') from None
             self.fds[segment] = (now, fd)
             return fd
 

src/borg/testsuite/repository.py

@@ -761,6 +761,15 @@ def corrupt_object(self, id_):
     def delete_segment(self, segment):
         self.repository.io.delete_segment(segment)
 
+    def delete_segment_dir(self, segment_dir):
+        shutil.rmtree(os.path.join(self.tmppath, 'repository', 'data', str(segment_dir)))
+
+    def set_segments_per_dir(self, segments_per_dir):
+        # note: only call this on a repository that does not have any segment files yet.
+        self.repository.config.set('repository', 'segments_per_dir', str(segments_per_dir))
+        self.repository.save_config(self.repository.path, self.repository.config)
+        self.reopen()
+
     def delete_index(self):
         os.unlink(os.path.join(self.tmppath, 'repository', f'index.{self.get_head()}'))
 
@@ -796,6 +805,31 @@ def test_repair_missing_segment(self):
         self.check(repair=True, status=True)
         self.assert_equal({1, 2, 3}, self.list_objects())
 
+    def test_missing_segment_file(self):
+        # a segment file the (still valid) repository index refers to is gone, e.g. because a
+        # filesystem check removed it. borg must tell what is wrong instead of crashing with a
+        # FileNotFoundError traceback, see #3225.
+        self.add_objects([[1, 2, 3], [4, 5, 6]])
+        self.delete_segment(2)  # the segment file with objects 4, 5, 6 in it
+        self.assert_raises(IntegrityError, lambda: self.get_objects(4))
+
+    def test_repair_missing_segment_dir(self):
+        # the segment dir borg needs to write a segment file into is gone, e.g. because a
+        # filesystem check removed it. borg must re-create it instead of crashing with a
+        # FileNotFoundError traceback, see #3225.
+        self.set_segments_per_dir(3)  # so the segments of this test are spread over multiple dirs
+        self.add_objects([[1, 2, 3], [4, 5, 6]])  # segments 0, 2: data, segments 1, 3: commits
+        # break the commit tag in segment 1 and remove data/1 (which has the commit segment 3 in
+        # it), so no valid commit is left and check --repair has to write a commit tag to segment
+        # 4 - which belongs into the removed data/1 dir.
+        with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd:
+            fd.seek(-1, os.SEEK_END)
+            fd.write(b'X')
+        self.delete_segment_dir(1)
+        self.check(repair=True, status=True)
+        self.check(status=True)
+        self.assert_equal({1, 2, 3, 4, 5, 6}, self.list_objects())
+
     def test_repair_missing_commit_segment(self):
         self.add_objects([[1, 2, 3], [4, 5, 6]])
         self.delete_segment(3)
@@ -1056,6 +1090,14 @@ def test_repair_missing_segment(self):
         # skip this test, files in RemoteRepository cannot be deleted
         pass
 
+    def test_missing_segment_file(self):
+        # skip this test, files in RemoteRepository cannot be deleted
+        pass
+
+    def test_repair_missing_segment_dir(self):
+        # skip this test, files in RemoteRepository cannot be deleted
+        pass
+
 
 class RemoteLoggerTestCase(BaseTestCase):
     def setUp(self):