diff --git a/doc/design/external_backend_protocol/comment_3_cd3410e6c1088289f71ed30b34d802f9._comment b/doc/design/external_backend_protocol/comment_3_cd3410e6c1088289f71ed30b34d802f9._comment
new file mode 100644
index 0000000000..12a7c7bb08
--- /dev/null
+++ b/doc/design/external_backend_protocol/comment_3_cd3410e6c1088289f71ed30b34d802f9._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="ah.nikfal@ad3e37b2c18d5aea546f662a0ba95796d0ef33ed"
+ nickname="ah.nikfal"
+ avatar="http://cdn.libravatar.org/avatar/0705936f3b4b8300018ce0ab62a114e5"
+ subject="How to use git-annex-backend-XFOO"
+ date="2022-12-09T12:58:57Z"
+ content="""
+After creating git-annex-backend-XFOO, how should I use this file? Where should I put this file?
+"""]]

diff --git a/Annex/Locations.hs b/Annex/Locations.hs
index ace33a5da2..e8361b377e 100644
--- a/Annex/Locations.hs
+++ b/Annex/Locations.hs
@@ -50,6 +50,7 @@ module Annex.Locations (
 	gitAnnexSmudgeLog,
 	gitAnnexSmudgeLock,
 	gitAnnexRestageLog,
+	gitAnnexRestageLogOld,
 	gitAnnexRestageLock,
 	gitAnnexMoveLog,
 	gitAnnexMoveLock,
@@ -385,6 +386,10 @@ gitAnnexSmudgeLock r = gitAnnexDir r P.</> "smudge.lck"
 gitAnnexRestageLog :: Git.Repo -> RawFilePath
 gitAnnexRestageLog r = gitAnnexDir r P.</> "restage.log"
 
+{- .git/annex/restage.old is used while restaging files in git -}
+gitAnnexRestageLogOld :: Git.Repo -> RawFilePath
+gitAnnexRestageLogOld r = gitAnnexDir r P.</> "restage.old"
+
 gitAnnexRestageLock :: Git.Repo -> RawFilePath
 gitAnnexRestageLock r = gitAnnexDir r P.</> "restage.lck"
 
diff --git a/CHANGELOG b/CHANGELOG
index 341bd930b2..e40910c576 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -7,6 +7,8 @@ git-annex (10.20221105) UNRELEASED; urgency=medium
   * When youtube-dl is not available in PATH, use yt-dlp instead.
   * Support parsing yt-dpl output to display download progress.
   * test: Add --test-debug option.
+  * Fix a hang that occasionally occurred during commands such as move.
+    (A bug introduced in 10.20220927)
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/Logs/File.hs b/Logs/File.hs
index 7609a40593..56b0c90dda 100644
--- a/Logs/File.hs
+++ b/Logs/File.hs
@@ -13,8 +13,10 @@ module Logs.File (
 	appendLogFile,
 	modifyLogFile,
 	streamLogFile,
+	streamLogFileUnsafe,
 	checkLogFile,
 	calcLogFile,
+	calcLogFileUnsafe,
 ) where
 
 import Annex.Common
@@ -98,7 +100,12 @@ checkLogFile f lck matchf = withSharedLock lck $ bracket setup cleanup go
 
 -- | Folds a function over lines of a log file to calculate a value.
 calcLogFile :: RawFilePath -> RawFilePath -> t -> (L.ByteString -> t -> t) -> Annex t
-calcLogFile f lck start update = withSharedLock lck $ bracket setup cleanup go
+calcLogFile f lck start update =
+	withSharedLock lck $ calcLogFileUnsafe f start update
+
+-- | Unsafe version that does not do locking.
+calcLogFileUnsafe :: RawFilePath -> t -> (L.ByteString -> t -> t) -> Annex t
+calcLogFileUnsafe f start update = bracket setup cleanup go
   where
 	setup = liftIO $ tryWhenExists $ openFile f' ReadMode
 	cleanup Nothing = noop
@@ -125,7 +132,15 @@ calcLogFile f lck start update = withSharedLock lck $ bracket setup cleanup go
 -- is running.
 streamLogFile :: FilePath -> RawFilePath -> Annex () -> (String -> Annex ()) -> Annex ()
 streamLogFile f lck finalizer processor = 
-	withExclusiveLock lck $ bracketOnError setup cleanup go
+	withExclusiveLock lck $ do
+		streamLogFileUnsafe f finalizer processor
+		liftIO $ writeFile f ""
+		setAnnexFilePerm (toRawFilePath f)
+
+-- Unsafe version that does not do locking, and does not empty the file
+-- at the end.
+streamLogFileUnsafe :: FilePath -> Annex () -> (String -> Annex ()) -> Annex ()
+streamLogFileUnsafe f finalizer processor = bracketOnError setup cleanup go
   where
 	setup = liftIO $ tryWhenExists $ openFile f ReadMode 
 	cleanup Nothing = noop
@@ -135,8 +150,6 @@ streamLogFile f lck finalizer processor =
 		mapM_ processor =<< liftIO (lines <$> hGetContents h)
 		liftIO $ hClose h
 		finalizer
-		liftIO $ writeFile f ""
-		setAnnexFilePerm (toRawFilePath f)
 
 createDirWhenNeeded :: RawFilePath -> Annex () -> Annex ()
 createDirWhenNeeded f a = a `catchNonAsync` \_e -> do
diff --git a/Logs/Restage.hs b/Logs/Restage.hs
index 260a2c53cb..5d4e2e0910 100644
--- a/Logs/Restage.hs
+++ b/Logs/Restage.hs
@@ -13,9 +13,11 @@ import Annex.Common
 import Git.FilePath
 import Logs.File
 import Utility.InodeCache
+import Annex.LockFile
 
 import qualified Data.ByteString as S
 import qualified Data.ByteString.Lazy as L
+import qualified Utility.RawFilePath as R
 
 -- | Log a file whose pointer needs to be restaged in git.
 -- The content of the file may not be a pointer, if it is populated with
@@ -27,31 +29,61 @@ writeRestageLog f ic = do
 	lckf <- fromRepo gitAnnexRestageLock
 	appendLogFile logf lckf $ L.fromStrict $ formatRestageLog f ic
 
--- | Streams the content of the restage log, and then empties the log at
--- the end.
+-- | Streams the content of the restage log.
 --
--- If the processor or finalizer is interrupted or throws an exception,
--- the log file is left unchanged.
+-- First, the content of the log file is moved to the restage.old file.
+-- If that file already exists, the content is appended, otherwise it's
+-- renamed to that.
+--
+-- The log file is kept locked during that, but the lock is then
+-- released. The processor may do something that itself needs to take the
+-- lock, so it's important that the lock not be held while running it.
+--
+-- The content of restage.old file is then streamed to the processor, 
+-- and then the finalizer is run, ending with emptying restage.old.
 --
--- Locking is used to prevent new items being added to the log while this
--- is running.
+-- If the processor or finalizer is interrupted or throws an exception,
+-- restage.old is left populated to be processed later.
 streamRestageLog :: Annex () -> (TopFilePath -> InodeCache -> Annex ()) -> Annex ()
 streamRestageLog finalizer processor = do
 	logf <- fromRepo gitAnnexRestageLog
+	oldf <- fromRepo gitAnnexRestageLogOld
+	let oldf' = fromRawFilePath oldf
 	lckf <- fromRepo gitAnnexRestageLock
-	streamLogFile (fromRawFilePath logf) lckf finalizer $ \l -> 
+	
+	withExclusiveLock lckf $ liftIO $
+		whenM (R.doesPathExist logf) $
+			ifM (R.doesPathExist oldf)
+				( do
+					h <- openFile oldf' AppendMode
+					hPutStr h =<< readFile (fromRawFilePath logf)
+					hClose h
+					liftIO $ removeWhenExistsWith R.removeLink logf
+				, moveFile logf oldf
+				)
+
+	streamLogFileUnsafe oldf' finalizer $ \l -> 
 		case parseRestageLog l of
 			Just (f, ic) -> processor f ic
 			Nothing -> noop
+	
+	liftIO $ removeWhenExistsWith R.removeLink oldf
 
+-- | Calculate over both the current restage log, and also over the old
+-- one if it had started to be processed but did not get finished due
+-- to an interruption.
 calcRestageLog :: t -> ((TopFilePath, InodeCache) -> t -> t) -> Annex t
 calcRestageLog start update = do
 	logf <- fromRepo gitAnnexRestageLog
+	oldf <- fromRepo gitAnnexRestageLogOld
 	lckf <- fromRepo gitAnnexRestageLock
-	calcLogFile logf lckf start $ \l v -> 
-		case parseRestageLog (decodeBL l) of
-			Just pl -> update pl v
-			Nothing -> v
+	withSharedLock lckf $ do
+		mid <- calcLogFileUnsafe logf start process
+		calcLogFileUnsafe oldf mid process
+  where
+	process l v = case parseRestageLog (decodeBL l) of
+		Just pl -> update pl v
+		Nothing -> v
 
 formatRestageLog :: TopFilePath -> InodeCache -> S.ByteString
 formatRestageLog f ic = encodeBS (showInodeCache ic) <> ":" <> getTopFilePath f
diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_9_230038504d75704baacd4d3ac750ee95._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_9_230038504d75704baacd4d3ac750ee95._comment
new file mode 100644
index 0000000000..e9fb22ed77
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_9_230038504d75704baacd4d3ac750ee95._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2022-12-08T18:07:47Z"
+ content="""
+I've developed a fix. Gonna re-run the reproducer long enough to be sure
+the bug is squashed..
+"""]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_8_b3f282a0a0d4c5f174cf6831dbf3014c._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_8_b3f282a0a0d4c5f174cf6831dbf3014c._comment
new file mode 100644
index 0000000000..39b6d3c854
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_8_b3f282a0a0d4c5f174cf6831dbf3014c._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2022-12-07T20:55:35Z"
+ content="""
+Oh interesting, the same process has restage.lck open twice.
+
+	lrwx------ 1 joey joey 64 Dec  7 14:20 /proc/1710650/fd/12 -> /home/joey/tmp/.t/3/main0/.git/annex/restage.lck
+	lrwx------ 1 joey joey 64 Dec  7 16:53 /proc/1710650/fd/16 -> /home/joey/tmp/.t/3/main0/.git/annex/restage.lck
+
+There are other processes that have the same restage.lck open too. 
+In the process tree below, pid 1710382 does, and so does pid 1710483.
+
+	1710368 pts/8    S+     0:00 sh -c git-annex test --fakessh -- 'localhost' 'git-annex-shell '"'"'p2pstdio'"'"' '"'"'/home/joey/tmp/.t/3/main0'"'"' '"'"'6e958d13-c3cb-4c78-a360-e14308368520'"'"' --uuid 3a4d8ec8-3f33-4b80-bd67-119b1c65fc88'
+	1710369 pts/8    Sl+    0:00  \_ git-annex test --fakessh -- localhost git-annex-shell 'p2pstdio' '/home/joey/tmp/.t/3/main0' '6e958d13-c3cb-4c78-a360-e14308368520' --uuid 3a4d8ec8-3f33-4b80-bd67-119b1c65fc88
+	1710380 pts/8    S+     0:00      \_ /bin/sh -c git-annex-shell 'p2pstdio' '/home/joey/tmp/.t/3/main0' '6e958d13-c3cb-4c78-a360-e14308368520' --uuid 3a4d8ec8-3f33-4b80-bd67-119b1c65fc88
+	1710382 pts/8    Sl+    0:00          \_ git-annex-shell p2pstdio /home/joey/tmp/.t/3/main0 6e958d13-c3cb-4c78-a360-e14308368520 --uuid 3a4d8ec8-3f33-4b80-bd67-119b1c65fc88
+	1710411 pts/8    S+     0:00              \_ git --git-dir=/home/joey/tmp/.t/3/main0/.git --work-tree=/home/joey/tmp/.t/3/main0 --literal-pathspecs cat-file --batch
+	1710476 pts/8    S+     0:00              \_ git --git-dir=/home/joey/tmp/.t/3/main0/.git --work-tree=/home/joey/tmp/.t/3/main0 --literal-pathspecs -c core.safecrlf=false update-index -q --refresh -z --stdin
+	1710479 pts/8    S+     0:00                  \_ /bin/sh -c git-annex filter-process git-annex filter-process
+	1710483 pts/8    Sl+    0:21                      \_ git-annex filter-process
+	1710504 pts/8    S+     0:00                          \_ git --git-dir=.git --work-tree=. --literal-pathspecs cat-file --batch-check=%(objectname) %(objecttype) %(objectsize)
+	1710506 pts/8    S+     0:00                          \_ git --git-dir=.git --work-tree=. --literal-pathspecs cat-file --batch
+	1710533 pts/8    S+     0:00                          \_ git --git-dir=.git --work-tree=. --literal-pathspecs hash-object -w --stdin-paths --no-filters
+	1710540 pts/8    S+     0:00                          \_ git --git-dir=.git --work-tree=. --literal-pathspecs cat-file --batch
+
+    64 Dec  7 17:01 /proc/1710382/fd/13 -> /home/joey/tmp/.t/3/main0/.git/annex/restage.lck
+    64 Dec  7 16:53 /proc/1710483/fd/11 -> /home/joey/tmp/.t/3/main0/.git/annex/restage.lck
+    64 Dec  7 16:53 /proc/1710483/fd/19 -> /home/joey/tmp/.t/3/main0/.git/annex/restage.lck
+
+(Strange that pid 1710368 is not a child process of the main git-annex test
+run. I'm unsure why that happens. )
+
+Stracing 1710483, it is also stuck on `fcntl F_SETLKW`.
+This looks like it may be the main deadlock, because the parent is waiting on
+`git update-index`, which is waiting on `git-annex filter-process`, which is
+stuck waiting to take the lock... which its parent is holding!
+
+(There are also several other process trees, running `git-annex test --fakessh`,
+that have gotten deparented from the main test run, and are hanging around.
+Those also have restage.lck open, though in their cases it is a since-deleted
+file.)
+
+I think this is a bug in restagePointerFiles, which uses 
+streamRestageLog while refreshIndex is running.
+The restage.log was added 2 months ago, in
+[[!commit 6a3bd283b8af53f810982e002e435c0d7c040c59]].
+
+I suspect that `git-annex filter-process` is running populatePointerFile via
+Command.Smudge.getMoveRaceRecovery, which needs to write to the restage.log.
+That would explain why the deadlock only happens sometimes, because it
+involves a race that triggers that code.
+
+Reading the whole
+restage.log into memory first would probably avoid the bug, but is not ideal.
+"""]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_7_d897d97bfe8f13431f8c5d92e61bbd3d._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_7_d897d97bfe8f13431f8c5d92e61bbd3d._comment
new file mode 100644
index 0000000000..f36245700e
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_7_d897d97bfe8f13431f8c5d92e61bbd3d._comment
@@ -0,0 +1,114 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2022-12-07T18:19:19Z"
+ content="""
+Very good news! Reproduced the hang on kite, after 500 iterations of test
+suite.
+
+	All 0 tests passed (0.01s)
+	Tests
+	  Repo Tests v10 adjusted unlocked branch
+	    Init Tests
+	      init:            OK (0.44s)
+	      add:             OK (0.81s)
+	    move (ssh remote):
+
+strace of the `git-annex move` process:
+
+	joey@kite:~/tmp>strace  -p 1710605 -f
+	strace: Process 1710605 attached with 6 threads
+	[pid 1710671] futex(0x7fab6801c398, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710614] futex(0x5a21968, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710613] restart_syscall(<... resuming interrupted read ...> <unfinished ...>
+	[pid 1710611] futex(0x4868648, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
+	[pid 1710610] futex(0x4868648, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710605] futex(0x5a2164c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710611] <... futex resumed>)      = 0
+	[pid 1710610] <... futex resumed>)      = -1 EAGAIN (Resource temporarily unavailable)
+	[pid 1710611] epoll_wait(3,  <unfinished ...>
+	[pid 1710610] futex(0x48685e0, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
+	[pid 1710611] <... epoll_wait resumed>[], 64, 0) = 0
+	[pid 1710610] <... futex resumed>)      = 0
+	[pid 1710611] epoll_wait(3,  <unfinished ...>
+	[pid 1710610] read(10,  <unfinished ...>
+	[pid 1710611] <... epoll_wait resumed>[], 64, 0) = 0
+	[pid 1710610] <... read resumed>"\344\16\0\0\0\0\0\0", 8) = 8
+	[pid 1710611] epoll_wait(3,  <unfinished ...>
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] write(9, "\377\0\0\0\0\0\0\0", 8 <unfinished ...>
+	[pid 1710613] <... restart_syscall resumed>) = 1
+	[pid 1710610] <... write resumed>)      = 8
+	[pid 1710613] read(9,  <unfinished ...>
+	[pid 1710610] read(10,  <unfinished ...>
+	[pid 1710613] <... read resumed>"\377\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710613] futex(0x5a21968, FUTEX_WAKE_PRIVATE, 1 <unfinished ...>
+	[pid 1710614] <... futex resumed>)      = 0
+	[pid 1710613] <... futex resumed>)      = 1
+	[pid 1710613] poll([{fd=7, events=POLLIN}, {fd=9, events=POLLIN}], 2, -1 <unfinished ...>
+	[pid 1710614] futex(0x5a21970, FUTEX_WAKE_PRIVATE, 1) = 0
+	[pid 1710614] sched_getaffinity(0, 128, [0, 1]) = 8
+	[pid 1710614] rt_sigprocmask(SIG_BLOCK, [HUP INT QUIT TERM CONT TSTP], [], 8) = 0
+	[pid 1710610] <... read resumed>"\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] read(10,  <unfinished ...>
+	[pid 1710614] rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
+	[pid 1710614] futex(0x5a2196c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710610] <... read resumed>"\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710610] futex(0x486864c, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY
+
+Strace of the child:
+
+	joey@kite:~/tmp>strace  -p 1710650 -f
+	strace: Process 1710650 attached with 6 threads
+	[pid 1710658] futex(0x51ef9ec, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710655] futex(0x51ef7ec, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710653] read(10,  <unfinished ...>
+	[pid 1710650] fcntl(12, F_SETLKW, {l_type=F_WRLCK, l_whence=SEEK_SET, l_start=0, l_len=0} <unfinished ...>
+	[pid 1710673] futex(0x7f6ec001c3e8, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710657] futex(0x7f6ed0000bf8, FUTEX_WAIT_BITSET_PRIVATE|FUTEX_CLOCK_REALTIME, 0, NULL, FUTEX_BITSET_MATCH_ANY <unfinished ...>
+	[pid 1710653] <... read resumed>"\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+	[pid 1710653] read(10, "\1\0\0\0\0\0\0\0", 8) = 8
+
+Which looks like it's blocked on taking a lock of
+`/home/joey/tmp/.t/3/main0/.git/annex/restage.lck`
+"""]]

diff --git a/Test/Framework.hs b/Test/Framework.hs
index d74ae7f3e1..9023eb0c83 100644
--- a/Test/Framework.hs
+++ b/Test/Framework.hs
@@ -749,7 +749,12 @@ parallelTestRunner' numjobs opts mkts
 	-- Make more parts than there are jobs, because some parts
 	-- are larger, and this allows the smaller parts to be packed
 	-- in more efficiently, speeding up the test suite overall.
-	numparts = numjobs * 2
+	--
+	-- When there is a pattern, splitting into parts will cause
+	-- extra work.
+	numparts = if haspattern
+		then 1
+		else numjobs * 2
 
 	worker rs nvar a = do
 		(n, m) <- atomically $ do
@@ -814,15 +819,16 @@ parallelTestRunner' numjobs opts mkts
 					, exitFailure
 					)
 	
-	tastyopts = case lookupOption (tastyOptionSet opts) of
+	(haspattern, tastyopts) = case lookupOption (tastyOptionSet opts) of
 		-- Work around limitation of tasty; when tests to run
 		-- are limited to a pattern, it does not include their
 		-- dependencies. So, add another pattern including the
 		-- init tests, which are a dependency of most tests.
 		TestPattern (Just p) -> 
-			setOption (TestPattern (Just (TP.Or p (TP.ERE initTestsName))))
-				(tastyOptionSet opts)
-		TestPattern Nothing -> tastyOptionSet opts
+			(True, setOption (TestPattern (Just (TP.Or p (TP.ERE initTestsName))))
+				(tastyOptionSet opts))
+		TestPattern Nothing ->
+			(False, tastyOptionSet opts)
 
 topLevelTestGroup :: [TestTree] -> TestTree
 topLevelTestGroup = testGroup "Tests"
diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
index fffca75e74..030f6816ee 100644
--- a/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
@@ -14,4 +14,6 @@ This will limit the test suite to only running the that test:
 
 I have that repeating locally in a loop, in case it's just something that
 happens every 1000 test runs or something like that.
+
+(I've patched git-annex to speed up the above command 400%.)
 """]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
index 5b507ba07e..fffca75e74 100644
--- a/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
@@ -3,5 +3,15 @@
  subject="""comment 6"""
  date="2022-12-07T16:36:44Z"
  content="""
-Search me. :-/
+Well, a specific filesystem or mount option that the script sets up, for
+example. 
+
+Or if it's something of that kind, it's also intermittent.
+
+This will limit the test suite to only running the that test:
+
+	git-annex test --pattern '/move (ssh remote)/ && /Repo Tests v8 unlocked/'
+
+I have that repeating locally in a loop, in case it's just something that
+happens every 1000 test runs or something like that.
 """]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
new file mode 100644
index 0000000000..5b507ba07e
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_6_d12ddfb48706454eba1bd552a681e5ba._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2022-12-07T16:36:44Z"
+ content="""
+Search me. :-/
+"""]]

diff --git a/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
index 641fe4c3e0..af43a59b26 100644
--- a/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
+++ b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
@@ -27,6 +27,8 @@ Hit **Add Internet Archive item**
 
 git-annex version 8.20210223
 
+linux (ubuntu)
+
 ### Please provide any additional information below.
 
 [[!format sh """

diff --git a/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
index 4055033aee..641fe4c3e0 100644
--- a/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
+++ b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
@@ -2,8 +2,8 @@
 
 When I hit **Add Internet Archive item**, I get:
 
-Internal Server Error
-Unexpected parameters: x-amz-interactive-priority x-amz-mediatype
+Internal Server Error  
+Unexpected parameters: x-amz-interactive-priority x-amz-mediatype  
 git-annex version 8.20210223
 
 
@@ -18,7 +18,7 @@ Choose **Internet Archive**
 
 On the _Adding an Internet Archive item_ page, enter access key, secret key
 
-choose Media Type software
+choose Media Type software  
 Item Name: commons-software
 
 Hit **Add Internet Archive item**

diff --git a/doc/users/dckc.mdwn b/doc/users/dckc.mdwn
new file mode 100644
index 0000000000..47ca2ab309
--- /dev/null
+++ b/doc/users/dckc.mdwn
@@ -0,0 +1 @@
+See: [MadMode: Dan Connolly's tinkering lab notebook](https://www.madmode.com/), [dckc on github](https://github.com/dckc)

diff --git a/doc/bugs/error_creating_internet_archive_item_via_web_UI/comment_1_4047d926154b444f33674f4db52be04d._comment b/doc/bugs/error_creating_internet_archive_item_via_web_UI/comment_1_4047d926154b444f33674f4db52be04d._comment
new file mode 100644
index 0000000000..6168e33d67
--- /dev/null
+++ b/doc/bugs/error_creating_internet_archive_item_via_web_UI/comment_1_4047d926154b444f33674f4db52be04d._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="dckc@4f72e5f6436280fd19a91a9805af71b9921a4f9b"
+ nickname="dckc"
+ avatar="http://cdn.libravatar.org/avatar/37c748efab0968b1203f146fd1d0e5ae"
+ subject="keep me posted by email please"
+ date="2022-12-07T14:40:52Z"
+ content="""
+I think I neglected to hit the relevant checkbox the 1st time.
+"""]]

diff --git a/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
new file mode 100644
index 0000000000..4055033aee
--- /dev/null
+++ b/doc/bugs/error_creating_internet_archive_item_via_web_UI.mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+
+When I hit **Add Internet Archive item**, I get:
+
+Internal Server Error
+Unexpected parameters: x-amz-interactive-priority x-amz-mediatype
+git-annex version 8.20210223
+
+
+### What steps will reproduce the problem?
+
+create a directory; git init; git annex init
+start the web ui; point it at said directory
+
+Hit the **+ Add another repository** button
+
+Choose **Internet Archive**
+
+On the _Adding an Internet Archive item_ page, enter access key, secret key
+
+choose Media Type software
+Item Name: commons-software
+
+Hit **Add Internet Archive item**
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version 8.20210223
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I recently started trying to use it to manage photos, but I'm struggling a bit to get my head around it.
+Today I'm trying to use it to archive my open source software downloads.
+
+

diff --git a/doc/users/nobodyinperson.mdwn b/doc/users/nobodyinperson.mdwn
new file mode 100644
index 0000000000..16ea100009
--- /dev/null
+++ b/doc/users/nobodyinperson.mdwn
@@ -0,0 +1 @@
+I use git-annex to manage my research data.

diff --git a/doc/thanks/list b/doc/thanks/list
index 32c82e341c..b1b0c033ef 100644
--- a/doc/thanks/list
+++ b/doc/thanks/list
@@ -130,3 +130,5 @@ k0ld,
 Johannes Söderlund, 
 Lawrence Brogan, 
 M K, 
+Yann Büchau, 
+unqueued, 

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_5_4e7be0dd1eb70af4790e2bcba7bb6ea1._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_5_4e7be0dd1eb70af4790e2bcba7bb6ea1._comment
new file mode 100644
index 0000000000..257a4a3453
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_5_4e7be0dd1eb70af4790e2bcba7bb6ea1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2022-12-05T18:08:02Z"
+ content="""
+what kind of a \"fault\" could it be? ;)
+"""]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_4_a8847e219a57f3cf41330745e6a67b10._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_4_a8847e219a57f3cf41330745e6a67b10._comment
new file mode 100644
index 0000000000..c7e6f15e3f
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_4_a8847e219a57f3cf41330745e6a67b10._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2022-12-05T17:48:52Z"
+ content="""
+The rerun did not get stuck. So it's not something directly the fault 
+of the script running `git-annex test`, I think.
+"""]]

diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_16_783ada9438a00c0a23357bea2f19d0a1._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_16_783ada9438a00c0a23357bea2f19d0a1._comment
new file mode 100644
index 0000000000..7a8c66fed3
--- /dev/null
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_16_783ada9438a00c0a23357bea2f19d0a1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 16"""
+ date="2022-12-05T17:29:11Z"
+ content="""
+<https://github.com/yesodweb/persistent/issues/1441> is having some useful
+discussion. There are changes that would speed it up 2x. The best path may
+be improving sqlite to optimise `insert_`.
+"""]]

diff --git a/doc/bugs/fsck_--json_incomplete_error_reporting/comment_1_979d52131fe94a9e800f939da88b369e._comment b/doc/bugs/fsck_--json_incomplete_error_reporting/comment_1_979d52131fe94a9e800f939da88b369e._comment
new file mode 100644
index 0000000000..0e33e6f007
--- /dev/null
+++ b/doc/bugs/fsck_--json_incomplete_error_reporting/comment_1_979d52131fe94a9e800f939da88b369e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="kanak@3c4f6e7d832d88751c617b25bdbac896417eb93b"
+ nickname="kanak"
+ avatar="http://cdn.libravatar.org/avatar/708121dfec06e554300b2a3a73a26818"
+ subject="comment 1"
+ date="2022-12-04T17:40:38Z"
+ content="""
+Not just limited to numcopies:
+
+{\"command\":\"fsck\",\"success\":true,\"key\":\"SHA256E-s119046--239da5a85ddf8c4071d8803a864a896d13e2a2fd65fd5684fc2f6dcaf264e875.jpg\",\"error-messages\":[],\"file\":\"12/03/chandrian_22:37:41.jpg\",\"note\":\"checksum...\",\"input
+\":[\"12/03/chandrian_22:37:41.jpg\"]} 
+  ** Based on the location log, 12/03/chandrian_23:05:41.jpg                                                                                                                                                        
+  ** was expected to be present, but its content is missing.  
+
+
+
+"""]]

diff --git a/doc/bugs/fsck_--json_incomplete_error_reporting.mdwn b/doc/bugs/fsck_--json_incomplete_error_reporting.mdwn
new file mode 100644
index 0000000000..f47b6e299a
--- /dev/null
+++ b/doc/bugs/fsck_--json_incomplete_error_reporting.mdwn
@@ -0,0 +1,52 @@
+### Please describe the problem.
+
+I ran git annex fsck --json. I have some files that are lacking numcopies. I get output in stderr, but no error details in the json body.
+
+For example:
+
+{"command":"fsck","success":false,"key":"SHA256E-s165540--ddcf7ce58593667e1b836e2a7f28a9f5227f3d9ba46cf8f98c7ab9dd26ef1896.jpg","error-messages":[],"file":"2022/12/04/chandrian_10:06:41.jpg","dead":[],"untrusted":[],"input":["2022/12/04/chandrian_10:06:41.jpg"]}
+  Only 2 of 5 trustworthy copies exist of 2022/12/04/chandrian_11:05:20.jpg
+  Back it up with git-annex copy.
+
+
+It would be great if error-messages contained all the details for failures. Thank you
+
+
+### What steps will reproduce the problem?
+
+* create a new repo
+* annex a file
+* set numcopies to 2 or whatever
+* git annex fsck --json
+
+
+### What version of git-annex are you using? On what operating system?
+
+I'm on Fedora 37.
+
+git annex version
+git-annex version: 10.20221103
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.6.4.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! Git annex is amazing and is managing over 10 TB of data across 5 git annexes and around 10 hard drives. No data loss this entire time -- over 8 years.
+

diff --git a/doc/bugs/blake3_hash_support.mdwn b/doc/bugs/blake3_hash_support.mdwn
index 41c40f8c4d..b1440d43fb 100644
--- a/doc/bugs/blake3_hash_support.mdwn
+++ b/doc/bugs/blake3_hash_support.mdwn
@@ -1,6 +1,6 @@
 This is a patch that seems to work for my personal use.
 BLAKE3 does support variable lengths, but my code does not implement support for anything other than 256-bit (32-byte) digests.
-I'm familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not.
+I'm not familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not.
 
 [[!format patch """
 From efa115d94d1a5a52574d5760c6e951ed3c518667 Mon Sep 17 00:00:00 2001

diff --git a/doc/bugs/blake3_hash_support.mdwn b/doc/bugs/blake3_hash_support.mdwn
new file mode 100644
index 0000000000..41c40f8c4d
--- /dev/null
+++ b/doc/bugs/blake3_hash_support.mdwn
@@ -0,0 +1,164 @@
+This is a patch that seems to work for my personal use.
+BLAKE3 does support variable lengths, but my code does not implement support for anything other than 256-bit (32-byte) digests.
+I'm familiar enough with the codebase to be sure whether adding variable length support later is a backwards compatibility hazard or not.
+
+[[!format patch """
+From efa115d94d1a5a52574d5760c6e951ed3c518667 Mon Sep 17 00:00:00 2001
+From: edef <edef@edef.eu>
+Date: Fri, 2 Dec 2022 12:16:44 +0000
+Subject: [PATCH] support BLAKE3
+
+This uses the blake3 package from Hackage, since cryptonite does not
+have BLAKE3 support yet.
+
+diff --git a/Backend/Hash.hs b/Backend/Hash.hs
+index 550d8fc6c..809a82599 100644
+--- a/Backend/Hash.hs
++++ b/Backend/Hash.hs
+@@ -27,8 +27,11 @@ import qualified Data.ByteString as S
+ import qualified Data.ByteString.Short as S (toShort, fromShort)
+ import qualified Data.ByteString.Char8 as S8
+ import qualified Data.ByteString.Lazy as L
++import Data.IORef
++import Control.Arrow
+ import Control.DeepSeq
+ import Control.Exception (evaluate)
++import qualified BLAKE3
+ 
+ data Hash
+ 	= MD5Hash
+@@ -40,6 +43,7 @@ data Hash
+ 	| Blake2bpHash HashSize
+ 	| Blake2sHash HashSize
+ 	| Blake2spHash HashSize
++	| Blake3Hash
+ 
+ cryptographicallySecure :: Hash -> Bool
+ cryptographicallySecure (SHA2Hash _) = True
+@@ -49,6 +53,7 @@ cryptographicallySecure (Blake2bHash _) = True
+ cryptographicallySecure (Blake2bpHash _) = True
+ cryptographicallySecure (Blake2sHash _) = True
+ cryptographicallySecure (Blake2spHash _) = True
++cryptographicallySecure Blake3Hash = True
+ cryptographicallySecure SHA1Hash = False
+ cryptographicallySecure MD5Hash = False
+ 
+@@ -63,6 +68,7 @@ hashes = concat
+ 	, map (Blake2bpHash . HashSize) [512]
+ 	, map (Blake2sHash . HashSize) [256, 160, 224]
+ 	, map (Blake2spHash . HashSize) [256, 224]
++	, [Blake3Hash]
+ 	, [SHA1Hash]
+ 	, [MD5Hash]
+ 	]
+@@ -99,6 +105,7 @@ hashKeyVariety (Blake2bHash size) he = Blake2bKey size he
+ hashKeyVariety (Blake2bpHash size) he = Blake2bpKey size he
+ hashKeyVariety (Blake2sHash size) he = Blake2sKey size he
+ hashKeyVariety (Blake2spHash size) he = Blake2spKey size he
++hashKeyVariety Blake3Hash he = Blake3Key he
+ 
+ {- A key is a hash of its contents. -}
+ keyValue :: Hash -> KeySource -> MeterUpdate -> Annex Key
+@@ -219,6 +226,7 @@ hasher (Blake2bHash hashsize) = blake2bHasher hashsize
+ hasher (Blake2bpHash hashsize) = blake2bpHasher hashsize
+ hasher (Blake2sHash hashsize) = blake2sHasher hashsize
+ hasher (Blake2spHash hashsize) = blake2spHasher hashsize
++hasher Blake3Hash = blake3Hasher
+ 
+ mkHasher :: HashAlgorithm h => (L.ByteString -> Digest h) -> Context h -> Hasher
+ mkHasher h c = (show . h, mkIncrementalVerifier c descChecksum . sameCheckSum)
+@@ -272,6 +280,27 @@ blake2spHasher (HashSize hashsize)
+ 	| hashsize == 224 = mkHasher blake2sp_224 blake2sp_224_context
+ 	| otherwise = error $ "unsupported BLAKE2SP size " ++ show hashsize
+ 
++blake3Hasher :: Hasher
++blake3Hasher = (hash, incremental) where
++	finalize :: BLAKE3.Hasher -> BLAKE3.Digest BLAKE3.DEFAULT_DIGEST_LEN
++	finalize = BLAKE3.finalize
++
++	hash :: L.ByteString -> String
++	hash = show . finalize . L.foldlChunks ((. pure) . BLAKE3.update) BLAKE3.hasher
++
++	incremental :: Key -> IO IncrementalVerifier
++	incremental k = do
++		v <- newIORef (Just (BLAKE3.hasher, 0))
++		return $ IncrementalVerifier
++			{ updateIncrementalVerifier = \b ->
++				modifyIORef' v . fmap $ flip BLAKE3.update [b] *** (fromIntegral (S.length b) +)
++			, finalizeIncrementalVerifier =
++				fmap (sameCheckSum k . show . finalize . fst) <$> readIORef v
++			, unableIncrementalVerifier = writeIORef v Nothing
++			, positionIncrementalVerifier = fmap snd <$> readIORef v
++			, descIncrementalVerifier = descChecksum
++			}
++
+ sha1Hasher :: Hasher
+ sha1Hasher = mkHasher sha1 sha1_context
+ 
+diff --git a/Types/Key.hs b/Types/Key.hs
+index 271723982..ea71f85ed 100644
+--- a/Types/Key.hs
++++ b/Types/Key.hs
+@@ -214,6 +214,7 @@ data KeyVariety
+ 	| Blake2bpKey HashSize HasExt
+ 	| Blake2sKey HashSize HasExt
+ 	| Blake2spKey HashSize HasExt
++	| Blake3Key HasExt
+ 	| SHA1Key HasExt
+ 	| MD5Key HasExt
+ 	| WORMKey
+@@ -247,6 +248,7 @@ hasExt (Blake2bKey _ (HasExt b)) = b
+ hasExt (Blake2bpKey _ (HasExt b)) = b
+ hasExt (Blake2sKey _ (HasExt b)) = b
+ hasExt (Blake2spKey _ (HasExt b)) = b
++hasExt (Blake3Key (HasExt b)) = b
+ hasExt (SHA1Key (HasExt b)) = b
+ hasExt (MD5Key (HasExt b)) = b
+ hasExt WORMKey = False
+@@ -262,6 +264,7 @@ sameExceptExt (Blake2bKey sz1 _) (Blake2bKey sz2 _) = sz1 == sz2
+ sameExceptExt (Blake2bpKey sz1 _) (Blake2bpKey sz2 _) = sz1 == sz2
+ sameExceptExt (Blake2sKey sz1 _) (Blake2sKey sz2 _) = sz1 == sz2
+ sameExceptExt (Blake2spKey sz1 _) (Blake2spKey sz2 _) = sz1 == sz2
++sameExceptExt (Blake3Key _) (Blake3Key _) = True
+ sameExceptExt (SHA1Key _) (SHA1Key _) = True
+ sameExceptExt (MD5Key _) (MD5Key _) = True
+ sameExceptExt _ _ = False
+@@ -275,6 +278,7 @@ formatKeyVariety v = case v of
+ 	Blake2bpKey sz e -> adde e (addsz sz "BLAKE2BP")
+ 	Blake2sKey sz e -> adde e (addsz sz "BLAKE2S")
+ 	Blake2spKey sz e -> adde e (addsz sz "BLAKE2SP")
++	Blake3Key e -> adde e "BLAKE3"
+ 	SHA1Key e -> adde e "SHA1"
+ 	MD5Key e -> adde e "MD5"
+ 	WORMKey -> "WORM"
+@@ -337,6 +341,8 @@ parseKeyVariety "BLAKE2SP224"  = Blake2spKey (HashSize 224) (HasExt False)
+ parseKeyVariety "BLAKE2SP224E" = Blake2spKey (HashSize 224) (HasExt True)
+ parseKeyVariety "BLAKE2SP256"  = Blake2spKey (HashSize 256) (HasExt False)
+ parseKeyVariety "BLAKE2SP256E" = Blake2spKey (HashSize 256) (HasExt True)
++parseKeyVariety "BLAKE3"       = Blake3Key (HasExt False)
++parseKeyVariety "BLAKE3E"      = Blake3Key (HasExt True)
+ parseKeyVariety "SHA1"         = SHA1Key (HasExt False)
+ parseKeyVariety "SHA1E"        = SHA1Key (HasExt True)
+ parseKeyVariety "MD5"          = MD5Key (HasExt False)
+diff --git a/git-annex.cabal b/git-annex.cabal
+index cd58a4ca3..7c251e33b 100644
+--- a/git-annex.cabal
++++ b/git-annex.cabal
+@@ -362,6 +362,7 @@ Executable git-annex
+    securemem,
+    crypto-api,
+    cryptonite (>= 0.23),
++   blake3,
+    memory,
+    deepseq,
+    split,
+diff --git a/stack.yaml b/stack.yaml
+index 7dbfb657a..936ee841b 100644
+--- a/stack.yaml
++++ b/stack.yaml
+@@ -25,3 +25,4 @@ extra-deps:
+ - base64-bytestring-1.0.0.3
+ - bencode-0.6.1.1
+ - http-client-0.7.9
++- blake3-0.2@sha256:d1146b9a51ccfbb0532780778b6d016a614e3d44c05d8c1923dde9a8be869045,2448
+"""]]

diff --git a/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
index c0873940e2..f721d7bf19 100644
--- a/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
+++ b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
@@ -87,7 +87,7 @@ file6
 file7
 file8
 file9
-⬆⬆⬆ This should only output file8 and file9 ⬆⬆⬆
+⬆⬆⬆ This should only output file4 through file8 ⬆⬆⬆
 ```
 
 Yann / @nobodyinperson

diff --git a/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
index 7c1d9b583e..c0873940e2 100644
--- a/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
+++ b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
@@ -89,3 +89,5 @@ file8
 file9
 ⬆⬆⬆ This should only output file8 and file9 ⬆⬆⬆
 ```
+
+Yann / @nobodyinperson

diff --git a/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
new file mode 100644
index 0000000000..7c1d9b583e
--- /dev/null
+++ b/doc/todo/--metadata_fieldname__62____61__VALUE_string_comparison.mdwn
@@ -0,0 +1,91 @@
+Thank you for `git-annex`, it's awesome!
+
+I recently figured I could add `git-annex metadata` to my research data files that contains the start and end date of timeseries data inside the files so a quick lookup by date range (”which files contain data in that time range”) is possible.
+
+This is possible when using numeric timestamps (e.g. unix timestamp like `1669981463`) but not with stringy dates (e.g. `2022-11-12T20:10:14+0200`) as `--metadata fieldname>=VALUE` does _numeric_ comparison.
+
+## Proposal: How about when `--metadata fieldname>=VALUE` falls back to string comparison when `VALUE` can't be parsed as a number?
+
+## Test case
+
+Consider this script `make-git-annex-dir-with-timestamps.sh`:
+
+```sh
+#/bin/sh
+fmt="$1";test -n "$fmt" || fmt="%FT%T%z"
+# make a new git annex repository
+d=git-annex-with-times-"$fmt";chmod +w -R "$d";rm -rf "$d";mkdir "$d";cd "$d"
+git init
+git annex init
+# create some files
+for i in `seq 1 9`;do echo "File $i" > "file$i";done
+git annex add .
+git commit -m "Add files"
+# add metadata to files
+for i in `seq 1 9`;do 
+    time_start="$(date -d"$((-20 + $i)) hours" +"$fmt")"
+    (set -x;git annex metadata --set time-start="$time_start" "file$i")
+    time_end="$(date -d"$((-10 + $i)) hours" +"$fmt")"
+    (set -x;git annex metadata --set time-end="$time_end" "file$i")
+done
+timerange_start="$(date -d "-16 hours -5 minutes" +"$fmt")"
+timerange_end="$(date -d "-12 hours +5 minutes" +"$fmt")"
+(
+set -x
+git annex find \
+    "-(" --metadata "time-start>=$timerange_start"  --and --metadata "time-start<=$timerange_end" "-)" \
+    --or \
+    "-(" --metadata "time-end>=$timerange_start"  --and --metadata "time-end<=$timerange_end" "-)"
+)
+echo "⬆⬆⬆ This should only output file4 through file8 ⬆⬆⬆"
+```
+
+Invoked with unix timestamps time format, it works as expected:
+
+```sh
+> ./make-git-annex-dir-with-timestamps.sh '%s'
+# ...
++ git annex find '-(' --metadata 'time-start>=1669923315' --and --metadata 'time-start<=1669938315' '-)' --or '-(' --metadata 'time-end>=1669923315' --and --metadata 'time-end<=1669938315' '-)'
+file4
+file5
+file6
+file7
+file8
+⬆⬆⬆ This should only output file4 through file8 ⬆⬆⬆
+```
+
+However, other stringy date formats match all files:
+
+```bash
+# typical ISO-ish time format
+> ./make-git-annex-dir-with-timestamps.sh "%FT%T%z"
+# ...
++ git annex find '-(' --metadata 'time-start>=2022-12-01T20:49:37+0100' --and --metadata 'time-start<=2022-12-02T00:59:37+0100' '-)' --or '-(' --metadata 'time-end>=2022-12-01T20:49:37+0100' --and --metadata 'time-end<=2022-12-02T00:59:37+0100' '-)'
+file1
+file2
+file3
+file4
+file5
+file6
+file7
+file8
+file9
+⬆⬆⬆ This should only output file4 through file8 ⬆⬆⬆
+```
+
+```sh
+# git-annex's own time format for 'FIELDNAME-lastchanged'
+> ./make-git-annex-dir-with-timestamps.sh "%Y-%m-%d@%H-%M-%S"
+# ...
++ git annex find '-(' --metadata 'time-start>=2022-12-01@20-38-04' --and --metadata 'time-start<=2022-12-02@00-38-04' '-)' --or '-(' --metadata 'time-end>=2022-12-01@20-38-04' --and --metadata 'time-end<=2022-12-02@00-38-04' '-)'
+file1
+file2
+file3
+file4
+file5
+file6
+file7
+file8
+file9
+⬆⬆⬆ This should only output file8 and file9 ⬆⬆⬆
+```

diff --git a/doc/forum/View_for_locally_existing_files/comment_2_e8d7672850518f4859643a0ee40e57e1._comment b/doc/forum/View_for_locally_existing_files/comment_2_e8d7672850518f4859643a0ee40e57e1._comment
new file mode 100644
index 0000000000..07b04eec80
--- /dev/null
+++ b/doc/forum/View_for_locally_existing_files/comment_2_e8d7672850518f4859643a0ee40e57e1._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="agschaid"
+ avatar="http://cdn.libravatar.org/avatar/7789d7511c5da25d71021be4ddb7fe18"
+ subject="comment 2"
+ date="2022-12-01T21:12:11Z"
+ content="""
+Thank you! That is simply perfect.
+"""]]

diff --git a/doc/forum/View_for_locally_existing_files/comment_1_8ede4c8f1ca48e34629c46a77ba01f11._comment b/doc/forum/View_for_locally_existing_files/comment_1_8ede4c8f1ca48e34629c46a77ba01f11._comment
new file mode 100644
index 0000000000..ed0a26d773
--- /dev/null
+++ b/doc/forum/View_for_locally_existing_files/comment_1_8ede4c8f1ca48e34629c46a77ba01f11._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Lukey"
+ avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b"
+ subject="comment 1"
+ date="2022-12-01T20:49:35Z"
+ content="""
+`git annex adjust --hide-missing`
+"""]]

diff --git a/doc/forum/View_for_locally_existing_files.mdwn b/doc/forum/View_for_locally_existing_files.mdwn
new file mode 100644
index 0000000000..31c92536de
--- /dev/null
+++ b/doc/forum/View_for_locally_existing_files.mdwn
@@ -0,0 +1,8 @@
+Hi,
+
+Is there a way to make a "view" that only shows me files that are locally existing (meaning: where the binary is present in the repo I am working on)?
+
+I had a look at documentation and the forum but I did not find anything fitting (I honestly assume that I overlooked it somewhere). To me views, vfilter and such do not appear to meet my needs since they are only working on metadata.
+
+Why do I need this?
+I am starting to manage my music collection with git annex. On most devices I only have a fraction of the globally available collection. I would like to see only those symlinks that actually lead to binary files. This way I would not to confuse music players with broken symlinks and know at a glance what I can actually listen to right now.

diff --git a/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn b/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn
index 41c3b24556..c7560928f6 100644
--- a/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn
+++ b/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn
@@ -14,7 +14,7 @@ pip install git+https://gitlab.com/nobodyinperson/git-annex-metadata-edit
 ## ✨ Features
 
 - Operate on multiple files (recursively)
-- overwrite a metadata field
+- overwrite/remove a metadata field
 - add/remove specific values from a field in one go
 
 ## 📟 📹 Screencast

diff --git a/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn b/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn
new file mode 100644
index 0000000000..41c3b24556
--- /dev/null
+++ b/doc/forum/Editing_Metadata_in_your___36__EDITOR.mdwn
@@ -0,0 +1,26 @@
+Hey everyone,
+
+I made a Python script that launches your `$EDITOR` (or `$VISUAL`) to conveniently edit git-annex metadata.
+
+Code is [on Gitlab](https://gitlab.com/nobodyinperson/git-annex-metadata-edit).
+
+## 📥 Installation
+
+```bash
+# Installation
+pip install git+https://gitlab.com/nobodyinperson/git-annex-metadata-edit
+```
+
+## ✨ Features
+
+- Operate on multiple files (recursively)
+- overwrite a metadata field
+- add/remove specific values from a field in one go
+
+## 📟 📹 Screencast
+
+[![asciicast](https://asciinema.org/a/541576.svg)](https://asciinema.org/a/541576?autoplay=1)
+
+Cheers, 👍
+
+Yann

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_3_2bb944b19503dcf90fdf55e5098a2c02._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_3_2bb944b19503dcf90fdf55e5098a2c02._comment
new file mode 100644
index 0000000000..ed0234223a
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_3_2bb944b19503dcf90fdf55e5098a2c02._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2022-11-29T22:25:42Z"
+ content="""
+> I have re-ran the command to see if the bug replicates..
+
+note: it was considerable amount of time (days?) for it to take there ;)  I made a copy `test_fs_testonly.py` where I removed all other \"benchmarks\" prior running `annex test` -- might get there faster if works so feel free to interrupt and rerun that one.  That code is old (circa 2014 ;)) , and I should do some face lift but haven't had a chance yet :-/
+"""]]

diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_2_419de3fe25c490b251b8a6a7a9b24ed6._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_2_419de3fe25c490b251b8a6a7a9b24ed6._comment
new file mode 100644
index 0000000000..5c38116c68
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_2_419de3fe25c490b251b8a6a7a9b24ed6._comment
@@ -0,0 +1,44 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-29T21:45:15Z"
+ content="""
+I examined this situation on the machine. I had to ctrl-z the process to
+get a shell. 
+
+Then I checked what the stdin of 3710605 was, and it was still 
+open, and was a pipe from 3710526. So that's why the p2pstdio process 
+was still running; that pipe should be closed when the parent git-annex
+is done, but the parent was apparently stuck on something.
+
+Then I straced 3710605 but it was suspended (oops). So I ran `fg`. This
+somehow unstuck everything! The test suite finished up very fast.
+
+Here is what it output for the test that had gotten stuck:
+
+	Tests
+	  Repo Tests v8 unlocked
+	    Init Tests
+	      init:            OK (2.54s)
+	      add:             OK (5.48s)
+	    move (ssh remote): FAIL (958561.16s)
+	      ./Test/Framework.hs:398:
+	      bad content in location log for foo key SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77 uuid UUID "c129397d-8209-40ea-8347-16a8c3fe69de"
+	      expected: True
+	       but got: False
+
+That seems like it must come from here in the test suite:
+
+        git_annex "move" ["--to", "origin", annexedfile] "move --to of file"
+        inmainrepo $ annexed_present annexedfile
+
+So it seems that the content was moved back to origin successfully
+(`annexed_present` checks that the object file is present before checking
+the location log), but that the location log didn't get updated. Need
+to check if that update would have been done by the `p2pstdio` process
+or the `move` process.
+
+Why would a SIGCONT have unstuck it I wonder?
+
+I have re-ran the command to see if the bug replicates..
+"""]]

diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_4_2cded724d1a4368dca475b89010c23e2._comment b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_4_2cded724d1a4368dca475b89010c23e2._comment
new file mode 100644
index 0000000000..e2e4b5abfa
--- /dev/null
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_4_2cded724d1a4368dca475b89010c23e2._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="cnjr2"
+ avatar="http://cdn.libravatar.org/avatar/f7e9654cc967c8815947b19829bfd746"
+ subject="comment 4"
+ date="2022-11-29T16:53:39Z"
+ content="""
+Thank you Joey - I wrote my previous comment as your answer was coming in. Much obliged!
+"""]]

diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_3_1f4bc975621d24e6e43d887693ada800._comment b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_3_1f4bc975621d24e6e43d887693ada800._comment
new file mode 100644
index 0000000000..a2c7877140
--- /dev/null
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_3_1f4bc975621d24e6e43d887693ada800._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="cnjr2"
+ avatar="http://cdn.libravatar.org/avatar/f7e9654cc967c8815947b19829bfd746"
+ subject="found related bug"
+ date="2022-11-29T16:51:44Z"
+ content="""
+I have searched around some more and found a related bug report. As per [your comment](https://git-annex.branchable.com/bugs/Wrong_backend_extension_in_files_with_multiple_dots/#comment-1e0885e62a9c22baae6300d2312c5163) what constitutes a file extension is determined by heuristics. So I guess my question would become: Would it be a bad idea to have long (and ugly) file extensions (i.e. `.numbers`) be recognized as as extensions? What would be situations where that would go wrong?
+"""]]

diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_2_2836fe9dfa7b5140df1ce893044c254b._comment b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_2_2836fe9dfa7b5140df1ce893044c254b._comment
new file mode 100644
index 0000000000..128336acf6
--- /dev/null
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_2_2836fe9dfa7b5140df1ce893044c254b._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-29T16:47:44Z"
+ content="""
+Also you might want to consider unlocking files,
+either generally or on the OSX systems with this problem.
+Only locked files use symlinks that cause problems for these programs.
+
+	git-annex adjust --unlock
+"""]]

diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn
index f6c811d813..fb3f414c04 100644
--- a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn
@@ -29,3 +29,5 @@ local repository version: 10
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 git annex makes me so happy. As I had mentioned in a previous post some years ago, it solves problems that I have not anticipated and teaches me new concepts that I then apply across my work. Thank you so much Joey! :bow:
+
+> [[notabug|done]] --[[Joey]]
diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_1_0e3e0966f06ded18ecfd41f8084b4658._comment b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_1_0e3e0966f06ded18ecfd41f8084b4658._comment
new file mode 100644
index 0000000000..78008ecc56
--- /dev/null
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped/comment_1_0e3e0966f06ded18ecfd41f8084b4658._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-29T16:44:07Z"
+ content="""
+What is an extension and what is not is ambiguous
+to a computer. Eg, the filename "mr.toadswildride"
+does not have an extension.
+
+So git-annex has a heuristic, that an extension has
+a maximum length of 4. Which works for the vast
+majority of extensions in use. You can override
+that to some other value:
+
+	git config annex.maxextensionlength 7
+"""]]

diff --git a/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn
new file mode 100644
index 0000000000..f6c811d813
--- /dev/null
+++ b/doc/bugs/long_file_extensions___40__e.g._.numbers__41___dropped.mdwn
@@ -0,0 +1,31 @@
+### Please describe the problem.
+
+Using the SHA256E backend should preserve file extensions. This works for most of my files, but not files with a `.numbers` extension. Without the extension in the filename, Numbers (Excel of Apple) can not open the file (as described elsewhere in the forums). Copying the file from the `.git/annex/objects` and re-adding the extension allows me to open the file again, so the file is intact.
+
+I think it might be that git annex does not recognize `.numbers` as an extension because of its length? 
+
+### What steps will reproduce the problem?
+
+Adding a file with `.numbers` extension (e.g. `git annex add bids.numbers`) will drop the extension in the `.git/annex/objects`
+
+[[!format sh """
+bids.numbers -> ../.git/annex/objects/Xp/xZ/SHA256E-s225502--a420fa2986ddd5e4fad39758e91bc753f5f1199a64c47c84b5bd5b9c288b66c2/SHA256E-s225502--a420fa2986ddd5e4fad39758e91bc753f5f1199a64c47c84b5bd5b9c288b66c2
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+git-annex version: 10.20221103
+build flags: Assistant Webapp Pairing FsEvents TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.30 DAV-1.3.4 feed-1.3.2.1 ghc-8.10.7 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: darwin x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git annex makes me so happy. As I had mentioned in a previous post some years ago, it solves problems that I have not anticipated and teaches me new concepts that I then apply across my work. Thank you so much Joey! :bow:

diff --git a/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__/comment_1_a02dd00950793814ea9617e0eec738ba._comment b/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__/comment_1_a02dd00950793814ea9617e0eec738ba._comment
new file mode 100644
index 0000000000..fc25b4eeb7
--- /dev/null
+++ b/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__/comment_1_a02dd00950793814ea9617e0eec738ba._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Spellbind8200"
+ avatar="http://cdn.libravatar.org/avatar/ca9df5665bccf2918715685e2ec40a09"
+ subject="Found unlock"
+ date="2022-11-29T09:05:43Z"
+ content="""
+Nvm, i just read detailed documentation on \" git annex adjust --unlock\" and that does seem to do the trick. 
+"""]]

diff --git a/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__.mdwn b/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__.mdwn
new file mode 100644
index 0000000000..d2586d8e17
--- /dev/null
+++ b/doc/forum/Prepare_simple_USB_backup___40__for_WAF__41__.mdwn
@@ -0,0 +1,5 @@
+I recently started using this awesome tool for backing up and location tracking personal data across multiple laptops, servers and usb drives. I want to create a simple backup archive on bunch of usb hdds so my family can have access to this data without having to learn any tool or me being around.
+
+I am able to do "rsync -PavLh" from my annex to usb drive and create a full simple backup. However, i lose location tracking with that since git-annex won't know what files are already copied. How do i prepare a simple exFat or NTFS usb with all data clearly visible without sim links or anything so that anyone can use it?
+
+Is it good idea to just create an annex on ntfs drive and then mark it unlocked before disconnecting? That way, however, connects that ntfs drive to a windows machine, they can just see directories without symlink shenanigans. Any other alternative? Any way to always mark a repo unlocked?

diff --git a/CHANGELOG b/CHANGELOG
index 78c8abc552..341bd930b2 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,7 @@ git-annex (10.20221105) UNRELEASED; urgency=medium
     large repository. Instead that scan is done on demand.
   * When youtube-dl is not available in PATH, use yt-dlp instead.
   * Support parsing yt-dpl output to display download progress.
+  * test: Add --test-debug option.
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/Test.hs b/Test.hs
index 6d3c58d3c5..89a947f755 100644
--- a/Test.hs
+++ b/Test.hs
@@ -92,7 +92,7 @@ import qualified Utility.Gpg
 
 optParser :: Parser TestOptions
 optParser = TestOptions
-	<$> snd (tastyParser (tests 1 False True (TestOptions mempty False False Nothing mempty mempty)))
+	<$> snd (tastyParser (tests 1 False True (TestOptions mempty False False Nothing mempty False mempty)))
 	<*> switch
 		( long "keep-failures"
 		<> help "preserve repositories on test failure"
@@ -111,6 +111,10 @@ optParser = TestOptions
 		<> help "run tests with a git config set"
 		<> metavar "NAME=VALUE"
 		))
+	<*> switch
+		( long "test-debug"
+		<> help "show debug messages for commands run by test suite"
+		)
 	<*> cmdParams "non-options are for internal use only"
   where
 	parseconfigvalue s = case break (== '=') s of
diff --git a/Test/Framework.hs b/Test/Framework.hs
index a4f24066ed..d74ae7f3e1 100644
--- a/Test/Framework.hs
+++ b/Test/Framework.hs
@@ -65,11 +65,20 @@ import qualified Command.Uninit
 
 -- Run a process. The output and stderr is captured, and is only
 -- displayed if the process does not return the expected value.
+--
+-- In debug mode, the output is allowed to pass through.
 testProcess :: String -> [String] -> Maybe [(String, String)] -> (Bool -> Bool) -> String -> Assertion
 testProcess command params environ expectedret faildesc = do
 	let p = (proc command params) { env = environ }
-	(transcript, ret) <- Utility.Process.Transcript.processTranscript' p Nothing
-	(expectedret ret) @? (faildesc ++ " failed (transcript follows)\n" ++ transcript)
+	debug <- testDebug . testOptions <$> getTestMode
+	if debug
+		then do
+			ret <- withCreateProcess p $ \_ _ _ pid ->
+				waitForProcess pid
+			(expectedret (ret == ExitSuccess)) @? (faildesc ++ " failed")
+		else do
+			(transcript, ret) <- Utility.Process.Transcript.processTranscript' p Nothing
+			(expectedret ret) @? (faildesc ++ " failed (transcript follows)\n" ++ transcript)
 
 -- Run git. (Do not use to run git-annex as the one being tested
 -- may not be in path.)
@@ -98,7 +107,11 @@ git_annex_shouldfail' = git_annex'' (== False)
 git_annex'' :: (Bool -> Bool) -> String -> [String] -> Maybe [(String, String)] -> String -> Assertion
 git_annex'' expectedret command params environ faildesc = do
 	pp <- Annex.Path.programPath
-	testProcess pp (command:params) environ expectedret faildesc
+	debug <- testDebug . testOptions <$> getTestMode
+	let params' = if debug
+		then "--debug":params
+		else params
+	testProcess pp (command:params') environ expectedret faildesc
 
 {- Runs git-annex and returns its standard output. -}
 git_annex_output :: String -> [String] -> IO String
diff --git a/Types/Test.hs b/Types/Test.hs
index 9d0af77208..90a182043f 100644
--- a/Types/Test.hs
+++ b/Types/Test.hs
@@ -19,6 +19,7 @@ data TestOptions = TestOptions
 	, fakeSsh :: Bool
 	, concurrentJobs :: Maybe Concurrency
 	, testGitConfig :: [(ConfigKey, ConfigValue)]
+	, testDebug :: Bool
 	, internalData :: CmdParams
 	}
 
diff --git a/doc/bugs/git_annex_test_never_exits__63__/comment_1_e07a5dd0abb2049b4bb0bf26fa2ebab5._comment b/doc/bugs/git_annex_test_never_exits__63__/comment_1_e07a5dd0abb2049b4bb0bf26fa2ebab5._comment
new file mode 100644
index 0000000000..0df28ac7b5
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__/comment_1_e07a5dd0abb2049b4bb0bf26fa2ebab5._comment
@@ -0,0 +1,51 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-28T18:11:41Z"
+ content="""
+Seems like the test it is stuck in is "move (ssh remote)".
+
+Is this replicable?
+
+I wonder if `test_fs.py` is doing something that causes the problem.
+There is some complexity in there. Maybe it's happening on one
+particular FS.
+
+If it happens when not run by `test_fs.py`, it would be useful to get a
+debug log, because I think that seeing the P2P protocol dump would help
+explain what has happened.
+
+There was not a good way to get the test suite to output a full debug
+log, so I had to implement one. With an updated build of git-annex,
+you can use:
+
+	git-annex test -J1 --test-debug
+
+Here's the expected output of that section of the test suite, when it
+is working properly:
+
+	move foo [2022-11-28 14:54:11.10011679] (Utility.Process) process [390399] chat: sh ["-c","git-annex test --fakessh -- 'localhost' 'git-annex-shell '\"'\"'p2pstdio'\"'\"' '\"'\"'/home/joey/src/git-annex/.t/3/main1'\"'\"' '\"'\"'--debug'\"'\"' '\"'\"'c76e0406-38ea-413e-9fb0-56cc32847734'\"'\"' --uuid 000c7285-ec19-4c15-9c67-4dd4b7c74775'"]
+	[2022-11-28 14:54:11.129754519] (P2P.IO) [ThreadId 4] P2P > AUTH-SUCCESS 000c7285-ec19-4c15-9c67-4dd4b7c74775
+	[2022-11-28 14:54:11.130259741] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P < AUTH-SUCCESS 000c7285-ec19-4c15-9c67-4dd4b7c74775
+	[2022-11-28 14:54:11.13041388] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P > VERSION 1
+	[2022-11-28 14:54:11.130680315] (P2P.IO) [ThreadId 4] P2P < VERSION 1
+	[2022-11-28 14:54:11.13087982] (P2P.IO) [ThreadId 4] P2P > VERSION 1
+	[2022-11-28 14:54:11.131127415] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P < VERSION 1
+	[2022-11-28 14:54:11.131262307] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P > CHECKPRESENT SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+	[2022-11-28 14:54:11.131679501] (P2P.IO) [ThreadId 4] P2P < CHECKPRESENT SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+	[2022-11-28 14:54:11.132388822] (P2P.IO) [ThreadId 4] P2P > FAILURE
+	[2022-11-28 14:54:11.13261351] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P < FAILURE
+	(to origin...) 
+	[2022-11-28 14:54:11.135583194] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P > PUT foo SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+	[2022-11-28 14:54:11.136134981] (P2P.IO) [ThreadId 4] P2P < PUT foo SHA256E-s20--e394a389d787383843decc5d3d99b6d184ffa5fddeec23b911f9ee7fc8b9ea77
+	[2022-11-28 14:54:11.136824468] (P2P.IO) [ThreadId 4] P2P > PUT-FROM 0
+	[2022-11-28 14:54:11.137123219] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P < PUT-FROM 0
+	...
+	[2022-11-28 14:54:11.148194515] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P > DATA 20
+	[2022-11-28 14:54:11.148414855] (P2P.IO) [ThreadId 4] P2P < DATA 20
+	[2022-11-28 14:54:11.148739395] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P > VALID
+	^M100%  20 B              2 KiB/s 0s[2022-11-28 14:54:11.152428942] (P2P.IO) [ThreadId 4] P2P < VALID
+	...
+	[2022-11-28 14:54:11.177599567] (P2P.IO) [ThreadId 4] P2P > SUCCESS
+	[2022-11-28 14:54:11.177858296] (P2P.IO) [ssh connection Just 390399] [ThreadId 4] P2P < SUCCESS
+"""]]
diff --git a/doc/git-annex-test.mdwn b/doc/git-annex-test.mdwn
index 17923cc026..18a219c6b2 100644
--- a/doc/git-annex-test.mdwn
+++ b/doc/git-annex-test.mdwn
@@ -44,6 +44,16 @@ framework. Pass --help for details about those.
   One valid use of this is to change a git configuration to a value that
   is planned to be the new default in a future version of git.
 
+* `--test-debug`
+
+  Normally output of commands run by the test suite is hidden, so even
+  when annex.debug or --debug is enabled, it will not be displayed.
+  This option makes the full output of commands run by the test suite be
+  displayed. It also makes the test suite run git-annex with --debug.
+
+  It's a good idea to use `-J1` in combinaton with this, otherwise
+  the output of concurrent tests will be mixed together.
+
 # SEE ALSO
 
 [[git-annex]](1)

diff --git a/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_1_a9501e2f347b957e7dc53a06fe59cff4._comment b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_1_a9501e2f347b957e7dc53a06fe59cff4._comment
new file mode 100644
index 0000000000..dadc7b49e4
--- /dev/null
+++ b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_1_a9501e2f347b957e7dc53a06fe59cff4._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-28T17:57:19Z"
+ content="""
+You can edit the hooks that git-annex installs however you need to. It will
+not overwrite modified hook files. If you edit this hook to have unix line
+endings, does the hook still work when using this repository on windows?
+
+Actually, I'm having difficulty seeing how the CR could have gotten into
+that hook. `mkHookScript` uses `unlines` which behaves the same on Windows
+as on Linux, so in either case lines should be separated only by `\n`.
+
+This makes me wonder if something else modified the hook after git-annex
+wrote it. If so, `git-annex init` should complain that the hook is
+modified.
+"""]]

diff --git a/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn
new file mode 100644
index 0000000000..b3e0f01375
--- /dev/null
+++ b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn
@@ -0,0 +1,62 @@
+### Please describe the problem.
+
+I have created a git-annex via Windows. If I use sshfs to mount this
+onto a Linux server, and then setup a new remote, I encounter the
+error:
+
+    fatal: cannot run hooks/post-receive: No such file or directory
+
+I've already identified that, due to the Windows created repository,
+the script uses crlf line-endings. And of course, the Linux kernel
+interprets the carriage-return as a trailing component of the shebang
+
+    #!/bin/sh^M
+
+and obviously can't find the executable.
+
+I've already worked around this by using a binfmt_misc approach
+workaround, registering the shebang with ^M to trigger a wrapper
+script. But I'm seeking the permanent fix of git-annex creating these
+hooks with unix line-endings only. I think it's fair to say, there's
+currently no approach that allows hooks to be in any other form than
+unix scripts, and I'm happy to stand corrected, that it makes little
+sense to have any other line-endings.
+
+### What steps will reproduce the problem?
+
+Run git annex sync against a Windows created remote git-annex, however
+that is mounted, on a Linux/unix host.
+
+### What version of git-annex are you using? On what operating system?
+
+    $ git annex version
+    git-annex version: 8.20210223
+
+on a Debian bullseye operating system.
+
+### Please provide any additional information below.
+
+[[!format sh """
+$ git annex sync
+On branch master
+Your branch is ahead of 'origin/master' by 1 commit.
+  (use "git push" to publish your local commits)
+
+
+It took 3.28 seconds to enumerate untracked files. 'status -uno'
+may speed it up, but you have to be careful not to forget to add
+new files yourself (see 'git help status').
+nothing to commit, working tree clean
+ok
+pull origin
+ok
+push origin
+fatal: cannot run hooks/post-receive: No such file or directory
+To /var/tmp/mnt/winhost-w10-5920/cygdrive/e/my.gitannex/
+   a0ebd1826d..15cfd91da4  git-annex -> synced/git-annex
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes. I think git-annex is a hidden gem of the open source
+community.

diff --git a/doc/bugs/git_annex_test_never_exits__63__.mdwn b/doc/bugs/git_annex_test_never_exits__63__.mdwn
new file mode 100644
index 0000000000..15fbce35c8
--- /dev/null
+++ b/doc/bugs/git_annex_test_never_exits__63__.mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+
+I got a new server for the repronim project (welcome `typhon` to the family of `smaug`, `drogon`, and `falkor`).  Wanted to redo https://www.datalad.org/test_fs_analysis.html benchmarking.  Added `git annex test` call to see how its time would vary across file systems etc.
+
+My surprise was that my script never exited.
+
+Current ps tree is  (yeah, running as `root`, likely will reinstall the thing anyways fully after I am done benchmarking)
+
+```
+root      185696  0.0  0.0  62536 54768 pts/1    S+   Nov15   2:56           python test_fs.py benchmark
+root     3655423  0.0  0.0   2484   516 pts/1    S+   Nov18   0:00             /bin/sh -c cd test9; git annex test
+root     3655424  0.0  0.0   9728  3340 pts/1    S+   Nov18   0:00               git annex test
+root     3655425  0.0  0.0 1076293924 28360 pts/1 Sl+ Nov18   0:00                 /usr/bin/git-annex test
+root     3696268  0.2  0.0 1074196636 41932 pts/1 Sl+ Nov18  13:14                   /usr/bin/git-annex --color=never test
+root     3710526  0.0  0.0 1074122856 44736 pts/1 Sl+ Nov18   0:00                     /usr/bin/git-annex move --to origin foo
+root     3710593  0.0  0.0      0     0 pts/1    Z+   Nov18   0:00                       [git] <defunct>
+root     3710594  0.0  0.0      0     0 pts/1    Z+   Nov18   0:00                       [git] <defunct>
+root     3710595  0.0  0.0      0     0 pts/1    Z+   Nov18   0:00                       [git] <defunct>
+root     3710596  0.0  0.0      0     0 pts/1    Z+   Nov18   0:00                       [git] <defunct>
+root     3710597  0.0  0.0   2484   516 pts/1    S+   Nov18   0:00                       sh -c git-annex test --fakessh -- 'localhost' 'git-annex-shell '"'"'p2pstdio'"'"' '"'"'/mnt/test/test9/.t/75/main0'"'"' '"'"'6129b8ec-34af-47d1-b7e6-3c4fcc0d33e3'"'"' --uuid c129397d-8209-40ea-8347-16a8c3fe69de'
+root     3710598  0.0  0.0 1074048960 20408 pts/1 Sl+ Nov18   0:00                         git-annex test --fakessh -- localhost git-annex-shell 'p2pstdio' '/mnt/test/test9/.t/75/main0' '6129b8ec-34af-47d1-b7e6-3c4fcc0d33e3' --uuid c129397d-8209-40ea-8347-16a8c3fe69de
+root     3710604  0.0  0.0   2484   580 pts/1    S+   Nov18   0:00                           /bin/sh -c git-annex-shell 'p2pstdio' '/mnt/test/test9/.t/75/main0' '6129b8ec-34af-47d1-b7e6-3c4fcc0d33e3' --uuid c129397d-8209-40ea-8347-16a8c3fe69de
+root     3710605  0.0  0.0 1074122860 18948 pts/1 Sl+ Nov18   2:39                             git-annex-shell p2pstdio /mnt/test/test9/.t/75/main0 6129b8ec-34af-47d1-b7e6-3c4fcc0d33e3 --uuid c129397d-8209-40ea-8347-16a8c3fe69de
+
+``` 
+
+
+### What version of git-annex are you using? On what operating system?
+
+```
+git-annex version: 10.20221003
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+
+```
+
+any ideas? need access to troubleshoot?
+
+[[!meta author=yoh]]
+[[!tag projects/repronim]]

diff --git a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
index 4eee65f34e..cc35f33847 100644
--- a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
+++ b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
@@ -7,3 +7,5 @@ The biggest problem is that there is no auto option, but in theory it’s possib
 I can try to code something, although I am not that proficient in Haskell, or run test code if needed on macOS 12. 
 
 -- Lena Wildervanck
+
+> [[dup|done]] --[[Joey]] 
diff --git a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS/comment_1_81bf4ac2fe810cc4e7a21c8bb4d0f15a._comment b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS/comment_1_81bf4ac2fe810cc4e7a21c8bb4d0f15a._comment
new file mode 100644
index 0000000000..cf619af016
--- /dev/null
+++ b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS/comment_1_81bf4ac2fe810cc4e7a21c8bb4d0f15a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-21T19:11:52Z"
+ content="""
+This todo aleady exists for the same thing:
+<https://git-annex.branchable.com/todo/support_macOS__39___cp_-c___40__cp_--reflink_equivalent__41__/>
+"""]]

diff --git a/Annex/YoutubeDl.hs b/Annex/YoutubeDl.hs
index e466f01eba..ba050d4024 100644
--- a/Annex/YoutubeDl.hs
+++ b/Annex/YoutubeDl.hs
@@ -249,8 +249,9 @@ youtubeDlOpts addopts = do
 	return (opts ++ addopts)
 
 youtubeDlCommand :: Annex String
-youtubeDlCommand = fromMaybe "youtube-dl" . annexYoutubeDlCommand 
-	<$> Annex.getGitConfig
+youtubeDlCommand = annexYoutubeDlCommand <$> Annex.getGitConfig >>= \case
+	Just c -> pure c
+	Nothing -> fromMaybe "yt-dlp" <$> liftIO (searchPath "youtube-dl")
 
 supportedScheme :: UrlOptions -> URLString -> Bool
 supportedScheme uo url = case parseURIRelaxed url of
diff --git a/CHANGELOG b/CHANGELOG
index d193ceca85..78fa628e4f 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -4,6 +4,7 @@ git-annex (10.20221105) UNRELEASED; urgency=medium
   * Sped up the initial scan for annexed files by 21%.
   * init: Avoid scanning for annexed files, which can be lengthy in a
     large repository. Instead that scan is done on demand.
+  * When youtube-dl is not available in PATH, use yt-dlp instead.
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/debian/control b/debian/control
index 762052186a..ac28ca029c 100644
--- a/debian/control
+++ b/debian/control
@@ -111,7 +111,7 @@ Recommends:
 	lsof,
 	gnupg,
 	bind9-host,
-	youtube-dl,
+	yt-dlp,
 	git-remote-gcrypt (>= 0.20130908-6),
 	nocache,
 	aria2,
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index 5eca6600c8..60811dce06 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -1719,8 +1719,8 @@ Remotes are configured using these settings in `.git/config`.
 
 * `annex.youtube-dl-options`
 
-  Options to pass to youtube-dl when using it to find the url to download
-  for a video.
+  Options to pass to youtube-dl (or yt-dlp) when using it to find the url
+  to download for a video.
 
   Some options may break git-annex's integration with youtube-dl. For
   example, the --output option could cause it to store files somewhere
@@ -1730,7 +1730,8 @@ Remotes are configured using these settings in `.git/config`.
 
 * `annex.youtube-dl-command`
 
-  Command to run for youtube-dl. Default is "youtube-dl".
+  Command to run for youtube-dl. Default is to use "youtube-dl" or 
+  if that is not available in the PATH, to use "yt-dlp".
 
 * `annex.aria-torrent-options`
 

diff --git a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
index e70a5b7e6c..4eee65f34e 100644
--- a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
+++ b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
@@ -5,3 +5,5 @@ I indeed discovered it was done for Linux after I posted it, but macOS with APFS
 The biggest problem is that there is no auto option, but in theory it’s possible to first try the clone option, and then fall back if it doesn’t work, although that would be inefficient. 
 
 I can try to code something, although I am not that proficient in Haskell, or run test code if needed on macOS 12. 
+
+-- Lena Wildervanck

diff --git a/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
new file mode 100644
index 0000000000..e70a5b7e6c
--- /dev/null
+++ b/doc/todo/Copy-on-Write__47__reflink__47__clonefile_support_on_macOS.mdwn
@@ -0,0 +1,7 @@
+Thank you for your previous message here: [[/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/]].
+
+I indeed discovered it was done for Linux after I posted it, but macOS with APFS also supports reflinks, under the `-c` (clonefile) option, which does the same as the `--reflink=always` option on Linux.
+
+The biggest problem is that there is no auto option, but in theory it’s possible to first try the clone option, and then fall back if it doesn’t work, although that would be inefficient. 
+
+I can try to code something, although I am not that proficient in Haskell, or run test code if needed on macOS 12. 

diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment
index 8348daadf0..379aafa328 100644
--- a/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment
@@ -13,11 +13,8 @@ something else. Weird!
 
 However, I also dumped the database to a sql script. Running that script
 with sqlite3 takes only 2.39 seconds. So, persistent-sqlite does have some
-performance overhead. My suspicion from profiling is that it's due to
-conversion between data types. First it builts up a Text containing a SQL
-statement (and not using a Builder). Then it uses encodeUtf8 on it to get
-a ByteString, and then it uses useAsCString. So there are 2 or 3 copies
-of the input data, which takes time and increases the allocations.
+performance overhead. Opened an issue: 
+<https://github.com/yesodweb/persistent/issues/1441>
 
 The test program:
 

diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment
new file mode 100644
index 0000000000..8348daadf0
--- /dev/null
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_15_145e29b42c46f15efc21c6d1ef9dd82d._comment
@@ -0,0 +1,65 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 15"""
+ date="2022-11-18T19:37:46Z"
+ content="""
+I wrote a standalone test program that uses persistant-sqlite to populate a
+database with the same data. It runs in 4.12 seconds. Which is sufficiently
+close to the 6 seconds I measured before for actual sqlite runtime in
+git-annex init.
+
+So, reconcileStaged is not slow only due to persistent-sqlite overhead, but
+something else. Weird!
+
+However, I also dumped the database to a sql script. Running that script
+with sqlite3 takes only 2.39 seconds. So, persistent-sqlite does have some
+performance overhead. My suspicion from profiling is that it's due to
+conversion between data types. First it builts up a Text containing a SQL
+statement (and not using a Builder). Then it uses encodeUtf8 on it to get
+a ByteString, and then it uses useAsCString. So there are 2 or 3 copies
+of the input data, which takes time and increases the allocations.
+
+The test program:
+
+	{-# LANGUAGE EmptyDataDecls             #-}
+	{-# LANGUAGE FlexibleContexts           #-}
+	{-# LANGUAGE GADTs                      #-}
+	{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+	{-# LANGUAGE MultiParamTypeClasses      #-}
+	{-# LANGUAGE OverloadedStrings          #-}
+	{-# LANGUAGE QuasiQuotes                #-}
+	{-# LANGUAGE TemplateHaskell            #-}
+	{-# LANGUAGE TypeFamilies               #-}
+	{-# LANGUAGE DerivingStrategies #-}
+	{-# LANGUAGE StandaloneDeriving #-}
+	{-# LANGUAGE UndecidableInstances #-}
+	{-# LANGUAGE DataKinds #-}
+	{-# LANGUAGE FlexibleInstances #-}
+	{-# LANGUAGE BangPatterns #-}
+	import           Control.Monad.IO.Class  (liftIO)
+	import           Database.Persist
+	import           Database.Persist.Sqlite
+	import           Database.Persist.TH
+	import           Data.Text
+	import           Data.Text.IO
+	import Control.Monad
+	
+	share [mkPersist sqlSettings, mkMigrate "migrateAll"] [persistLowerCase|
+	Val
+	    foo Text
+	    bar Text
+	    FooBarIndex foo bar
+	    BarFooIndex bar foo
+	|]
+	
+	main :: IO ()
+	main = runSqlite "sqlitedb" $ do
+	    runMigration migrateAll
+	    forM_ [1..(172156/2)] $ \x -> do
+	    	f <- liftIO Data.Text.IO.getLine
+	    	b <- liftIO Data.Text.IO.getLine
+	    	_ <- insert $ Val f b
+		return ()
+
+(Feed on stdin a series of pairs of lines of key and filename.)
+"""]]

diff --git a/Annex/Init.hs b/Annex/Init.hs
index 94f3ed5079..8b86572bec 100644
--- a/Annex/Init.hs
+++ b/Annex/Init.hs
@@ -37,7 +37,6 @@ import Types.RepoVersion
 import Annex.Version
 import Annex.Difference
 import Annex.UUID
-import Annex.WorkTree
 import Annex.Fixup
 import Annex.Path
 import Config
@@ -102,8 +101,8 @@ genDescription Nothing = do
 		Right username -> [username, at, hostname, ":", reldir]
 		Left _ -> [hostname, ":", reldir]
 
-initialize :: Bool -> Maybe String -> Maybe RepoVersion -> Annex ()
-initialize autoinit mdescription mversion = checkInitializeAllowed $ \initallowed -> do
+initialize :: Maybe String -> Maybe RepoVersion -> Annex ()
+initialize mdescription mversion = checkInitializeAllowed $ \initallowed -> do
 	{- Has to come before any commits are made as the shared
 	 - clone heuristic expects no local objects. -}
 	sharedclone <- checkSharedClone
@@ -113,7 +112,7 @@ initialize autoinit mdescription mversion = checkInitializeAllowed $ \initallowe
 	ensureCommit $ Annex.Branch.create
 
 	prepUUID
-	initialize' autoinit mversion initallowed
+	initialize' mversion initallowed
 	
 	initSharedClone sharedclone
 	
@@ -125,8 +124,8 @@ initialize autoinit mdescription mversion = checkInitializeAllowed $ \initallowe
 
 -- Everything except for uuid setup, shared clone setup, and initial
 -- description.
-initialize' :: Bool -> Maybe RepoVersion -> InitializeAllowed -> Annex ()
-initialize' autoinit mversion _initallowed = do
+initialize' :: Maybe RepoVersion -> InitializeAllowed -> Annex ()
+initialize' mversion _initallowed = do
 	checkLockSupport
 	checkFifoSupport
 	checkCrippledFileSystem
@@ -143,8 +142,6 @@ initialize' autoinit mversion _initallowed = do
 	unlessM isBareRepo $ do
 		hookWrite postCheckoutHook
 		hookWrite postMergeHook
-		unless autoinit $
-			scanAnnexedFiles
 
 	AdjustedBranch.checkAdjustedClone >>= \case
 		AdjustedBranch.InAdjustedClone -> return ()
@@ -206,7 +203,7 @@ ensureInitialized remotelist = getInitializedVersion >>= maybe needsinit checkUp
   where
 	needsinit = ifM autoInitializeAllowed
 		( do
-			tryNonAsync (initialize True Nothing Nothing) >>= \case
+			tryNonAsync (initialize Nothing Nothing) >>= \case
 				Right () -> noop
 				Left e -> giveup $ show e ++ "\n" ++
 					"git-annex: automatic initialization failed due to above problems"
@@ -259,7 +256,7 @@ autoInitialize remotelist = getInitializedVersion >>= maybe needsinit checkUpgra
   where
 	needsinit =
 		whenM (initializeAllowed <&&> autoInitializeAllowed) $ do
-			initialize True Nothing Nothing
+			initialize Nothing Nothing
 			autoEnableSpecialRemotes remotelist
 
 {- Checks if a repository is initialized. Does not check version for ugrade. -}
diff --git a/Assistant/MakeRepo.hs b/Assistant/MakeRepo.hs
index 632c4abda5..bad4951b1d 100644
--- a/Assistant/MakeRepo.hs
+++ b/Assistant/MakeRepo.hs
@@ -85,7 +85,7 @@ initRepo False _ dir desc mgroup = inDir dir $ do
 
 initRepo' :: Maybe String -> Maybe StandardGroup -> Annex ()
 initRepo' desc mgroup = unlessM isInitialized $ do
-	initialize False desc Nothing
+	initialize desc Nothing
 	u <- getUUID
 	maybe noop (defaultStandardGroup u) mgroup
 	{- Ensure branch gets committed right away so it is
diff --git a/CHANGELOG b/CHANGELOG
index a8b06f1412..d193ceca85 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,7 +1,9 @@
 git-annex (10.20221105) UNRELEASED; urgency=medium
 
   * Support quettabyte and yottabyte.
-  * Sped up the initial scanning for annexed files by 21%.
+  * Sped up the initial scan for annexed files by 21%.
+  * init: Avoid scanning for annexed files, which can be lengthy in a
+    large repository. Instead that scan is done on demand.
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/Command/ConfigList.hs b/Command/ConfigList.hs
index d259b87409..bb33f7102b 100644
--- a/Command/ConfigList.hs
+++ b/Command/ConfigList.hs
@@ -47,7 +47,7 @@ findOrGenUUID = do
 		else ifM (Annex.Branch.hasSibling <||> (isJust <$> Fields.getField Fields.autoInit))
 			( do
 				liftIO checkNotReadOnly
-				initialize True Nothing Nothing
+				initialize Nothing Nothing
 				getUUID
 			, return NoUUID
 			)
diff --git a/Command/Init.hs b/Command/Init.hs
index 8670ea221e..69e7516472 100644
--- a/Command/Init.hs
+++ b/Command/Init.hs
@@ -75,7 +75,7 @@ perform os = do
 			Just v | v /= wantversion ->
 				giveup $ "This repository is already a initialized with version " ++ show (fromRepoVersion v) ++ ", not changing to requested version."
 			_ -> noop
-	initialize False
+	initialize
 		(if null (initDesc os) then Nothing else Just (initDesc os))
 		(initVersion os)
 	unless (noAutoEnable os)
diff --git a/Command/Reinit.hs b/Command/Reinit.hs
index 42d47a0141..d11d807ab4 100644
--- a/Command/Reinit.hs
+++ b/Command/Reinit.hs
@@ -35,6 +35,6 @@ perform s = do
 		then return $ toUUID s
 		else Remote.nameToUUID s
 	storeUUID u
-	checkInitializeAllowed $ initialize' False Nothing
+	checkInitializeAllowed $ initialize' Nothing
 	Annex.SpecialRemote.autoEnable
 	next $ return True
diff --git a/Command/Upgrade.hs b/Command/Upgrade.hs
index f2f05122fc..77f569a15b 100644
--- a/Command/Upgrade.hs
+++ b/Command/Upgrade.hs
@@ -45,6 +45,6 @@ start (UpgradeOptions { autoOnly = True }) =
 start _ =
 	starting "upgrade" (ActionItemOther Nothing) (SeekInput []) $ do
 		whenM (isNothing <$> getVersion) $ do
-			initialize False Nothing Nothing
+			initialize Nothing Nothing
 		r <- upgrade False latestVersion
 		next $ return r
diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment
index 8c6505d068..eedca77d66 100644
--- a/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment
@@ -5,4 +5,15 @@
  content="""
 Implemented the two optimisations discussed above, and init in that
 repository dropped from 24 seconds to 19 seconds, a 21% speedup.
+
+I think that's as fast as reconcileStaged is likely to get without
+some deep optimisation of the persistent library.
+
+Then I realized that `git-annex init` does not really need to scan for
+associated files. That can be done later, when running a command that needs
+to access the keys database. Indeed, when git-annex is used in a clone of
+an annexed repo without explicitly running `git-annex init`, that's what
+it already did. I've implemented that, so now `git-annex init` takes 3
+seconds or so. The price will be paid later, the first time running a
+`git-annex add` or `git-annex unlock` or `git-annex get`.
 """]]

diff --git a/CHANGELOG b/CHANGELOG
index eed419b50f..a8b06f1412 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,7 +1,7 @@
 git-annex (10.20221105) UNRELEASED; urgency=medium
 
   * Support quettabyte and yottabyte.
-  * Sped up the initial scanning for annexed files by 15%.
+  * Sped up the initial scanning for annexed files by 21%.
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/Database/Keys/SQL.hs b/Database/Keys/SQL.hs
index 54e1a59bf7..77afb871a1 100644
--- a/Database/Keys/SQL.hs
+++ b/Database/Keys/SQL.hs
@@ -73,8 +73,8 @@ newtype WriteHandle = WriteHandle H.DbQueue
 queueDb :: SqlPersistM () -> WriteHandle -> IO ()
 queueDb a (WriteHandle h) = H.queueDb h checkcommit a
   where
-	-- commit queue after 1000 changes
-	checkcommit sz _lastcommittime = pure (sz > 1000)
+	-- commit queue after 10000 changes
+	checkcommit sz _lastcommittime = pure (sz > 10000)
 
 -- Insert the associated file.
 -- When the file was associated with a different key before,
diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
index 02dac893e3..36474beabe 100644
--- a/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
@@ -12,6 +12,4 @@ This will need some care to be implemented safely...
 
 I benchmarked it, and using insertUnique is no faster, but using insert is.
 This would be a 15% speed up.
-
-Update: Implemented this optimisation.
 """]]
diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment
new file mode 100644
index 0000000000..8c6505d068
--- /dev/null
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_14_8c3b13806adb731435b346a64990527b._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 14"""
+ date="2022-11-18T17:26:03Z"
+ content="""
+Implemented the two optimisations discussed above, and init in that
+repository dropped from 24 seconds to 19 seconds, a 21% speedup.
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index c4f81d8a80..eed419b50f 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,7 @@
 git-annex (10.20221105) UNRELEASED; urgency=medium
 
   * Support quettabyte and yottabyte.
+  * Sped up the initial scanning for annexed files by 15%.
 
  -- Joey Hess <id@joeyh.name>  Fri, 18 Nov 2022 12:58:06 -0400
 
diff --git a/Database/Keys.hs b/Database/Keys.hs
index 45f8d2f851..9e1043edae 100644
--- a/Database/Keys.hs
+++ b/Database/Keys.hs
@@ -132,10 +132,10 @@ openDb forwrite _ = do
 		let db = dbdir P.</> "db"
 		dbexists <- liftIO $ R.doesPathExist db
 		case dbexists of
-			True -> open db
+			True -> open db False
 			False -> do
 				initDb db SQL.createTables
-				open db
+				open db True
   where
 	-- If permissions don't allow opening the database, and it's being
 	-- opened for read, treat it as if it does not exist.
@@ -143,9 +143,9 @@ openDb forwrite _ = do
 		| forwrite = throwM e
 		| otherwise = return DbUnavailable
 	
-	open db = do
+	open db dbisnew = do
 		qh <- liftIO $ H.openDbQueue db SQL.containedTable
-		tc <- reconcileStaged qh
+		tc <- reconcileStaged dbisnew qh
 		return $ DbOpen (qh, tc)
 
 {- Closes the database if it was open. Any writes will be flushed to it.
@@ -260,8 +260,8 @@ isInodeKnown i s = or <$> runReaderIO ContentTable
  - So when using getAssociatedFiles, have to make sure the file still
  - is an associated file.
  -}
-reconcileStaged :: H.DbQueue -> Annex DbTablesChanged
-reconcileStaged qh = ifM (Git.Config.isBare <$> gitRepo)
+reconcileStaged :: Bool -> H.DbQueue -> Annex DbTablesChanged
+reconcileStaged dbisnew qh = ifM (Git.Config.isBare <$> gitRepo)
 	( return mempty
 	, do
 		gitindex <- inRepo currentIndexFile
@@ -384,7 +384,7 @@ reconcileStaged qh = ifM (Git.Config.isBare <$> gitRepo)
 						Nothing -> return False
 				send mdfeeder (Ref dstsha) $ \case
 					Just key -> do
-						liftIO $ SQL.addAssociatedFile key
+						liftIO $ addassociatedfile key
 							(asTopFilePath file)
 							(SQL.WriteHandle qh)
 						when (dstmode /= fmtTreeItemType TreeSymlink) $
@@ -497,6 +497,18 @@ reconcileStaged qh = ifM (Git.Config.isBare <$> gitRepo)
 	largediff :: Int
 	largediff = 1000
 
+	-- When the database is known to have been newly created and empty
+	-- before reconcileStaged started, it is more efficient to use 
+	-- newAssociatedFile. It's safe to use it here because this is run
+	-- with a lock held that blocks any other process that opens the
+	-- database, and when the database is newly created, there is no
+	-- existing process that has it open already. And it's not possible
+	-- for reconcileStaged to call this twice on the same filename with
+	-- two different keys.
+	addassociatedfile
+		| dbisnew = SQL.newAssociatedFile
+		| otherwise = SQL.addAssociatedFile
+
 {- Normally the keys database is updated incrementally when opened,
  - by reconcileStaged. Calling this explicitly allows running the
  - update at an earlier point.
diff --git a/Database/Keys/SQL.hs b/Database/Keys/SQL.hs
index cab4c58759..54e1a59bf7 100644
--- a/Database/Keys/SQL.hs
+++ b/Database/Keys/SQL.hs
@@ -1,6 +1,6 @@
 {- Sqlite database of information about Keys
  -
- - Copyright 2015-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -88,6 +88,15 @@ addAssociatedFile k f = queueDb $
   where
 	af = SFilePath (getTopFilePath f)
 
+-- Faster than addAssociatedFile, but only safe to use when the file
+-- was not associated with a different key before, as it does not delete
+-- any old key.
+newAssociatedFile :: Key -> TopFilePath -> WriteHandle -> IO ()
+newAssociatedFile k f = queueDb $
+	void $ insert $ Associated k af
+  where
+	af = SFilePath (getTopFilePath f)
+
 {- Note that the files returned were once associated with the key, but
  - some of them may not be any longer. -}
 getAssociatedFiles :: Key -> ReadHandle -> IO [TopFilePath]
diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
index 36474beabe..02dac893e3 100644
--- a/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
@@ -12,4 +12,6 @@ This will need some care to be implemented safely...
 
 I benchmarked it, and using insertUnique is no faster, but using insert is.
 This would be a 15% speed up.
+
+Update: Implemented this optimisation.
 """]]

diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_12_603d7228668a36dc848e706cdff64035._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_12_603d7228668a36dc848e706cdff64035._comment
new file mode 100644
index 0000000000..cd013fdacb
--- /dev/null
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_12_603d7228668a36dc848e706cdff64035._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2022-11-17T17:43:27Z"
+ content="""
+With --debug, it shows when it enters and leaves commitDb.
+
+commitDb is run around 70 times in this repository. Making it queue
+more changes into each commitDb, so it runs an order of magnitude fewer,
+did speed it up by around 1 second (from 24).
+
+Summing up the time spent in commitDb according to --debug, it is around 6.5
+seconds, out of the 24 second run. That is close to the actual time sqlite is
+taking.
+
+But per my comment above, with commitDb stubbed out, the whole run takes
+only 6 seconds. (I've re-verified that to be sure.) Why is it so much faster then?
+After doing some profiling and looking at flame graphs, I think it comes 
+down to allocations:
+
+(For future reference, here is a flame graph measuring runtimes
+of unmodified git-annex:  <https://tmp.joeyh.name/git-annex-flame-1.svg>  
+Shows the majority of time in commitDb as expected, and that it's due to 
+addAssociatedFile. The resulting upsert spends about 1/4th of its time reading
+and 3/4 writing. All unsurprising.. But also a sizable amount of time is
+spent in GC, and in IDLE, whatever that is.)
+
+Without it stubbed out:
+
+        total alloc = 6,385,408,240 bytes  (excludes profiling overheads)
+
+With it stubbed out:
+
+        total alloc = 1,714,898,200 bytes  (excludes profiling overheads)
+
+Here is a flame graph measuring allocations, not runtime, of unmodified git-annex:  
+https://tmp.joeyh.name/git-annex-flame-3.svg>  
+By far the main allocation is by mkSqlBackend, which is surprising because
+that represents a database handle. Why would it need to spend tons of memory
+making database handles?
+
+Starting to think this could be a bug in persistent or persistent-sqlite. I've
+tried disabling the lowlevel persistent parts of Database.Handle in case there
+was a bug in there, and using persistent more typically, but that didn't change
+the mkSqlBackend allocations. Next step might be a simpler test case.
+
+OTOH, it may be that the way that persistent uses the SqlBackend as a dictionary
+of functions, that calls to individual functions in it confuse the profiler,
+and allocation done to build SQL statements is being misattributed?
+"""]]
diff --git a/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
new file mode 100644
index 0000000000..36474beabe
--- /dev/null
+++ b/doc/bugs/performance_regression__63___init_takes_times_more/comment_13_a79fcbe80060d11582989a9fc31d4a92._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 13"""
+ date="2022-11-17T20:20:16Z"
+ content="""
+An idea from looking at the flame graphs, is that if
+reconcileStaged knew it was run on an empty keys db (or was doing a full
+scan of the tree, not an incremental updte), it could avoid doing the
+upsert, and do a blind insert instead.
+
+This will need some care to be implemented safely... 
+
+I benchmarked it, and using insertUnique is no faster, but using insert is.
+This would be a 15% speed up.
+"""]]

diff --git a/doc/tips/unlocked_files.mdwn b/doc/tips/unlocked_files.mdwn
index d8f1c9aec6..e530f4edf3 100644
--- a/doc/tips/unlocked_files.mdwn
+++ b/doc/tips/unlocked_files.mdwn
@@ -108,7 +108,7 @@ detail in [[todo/git_smudge_clean_interface_suboptiomal]].)
 ## using less disk space
 
 Unlocked files are handy, but they have one significant disadvantage
-compared with locked files: They use more disk space.
+compared with locked files: On most filesystems, they use more disk space.
 
 While only one copy of a locked file has to be stored, often
 two copies of an unlocked file are stored on disk. One copy is in
@@ -119,7 +119,10 @@ The reason for that second copy is to preserve the old version of the file,
 when you modify the unlocked file in the work tree. Being able to access
 old versions of files is an important part of git after all!
 
-That's a good safe default. But there are ways to use git-annex that
+(Some filesystems including btrfs and xfs support reflinks, and on those,
+the extra copy is a reflink, and takes up no additional space.)
+
+So two copies is a good safe default. But there are ways to use git-annex that
 make the second copy not be worth keeping:
 
 * When you're using git-annex to sync the current version of files across
diff --git a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn
index 72b8bf7330..bc49a7512f 100644
--- a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn
+++ b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn
@@ -1 +1,3 @@
 Some filesystems, like BTRFS or ZFS have copy on write support, which means that if files are copied in a special way they share sectors until one of the files is edited. Implementing support would make files take up less space if they are in unlocked mode and not significantly edited.
+
+> Already [[done]]. I did add a mention of it to the tip. --[[Joey]]

diff --git a/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_2_0a9372f3dba83b6cee3a33933bf2b006._comment b/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_2_0a9372f3dba83b6cee3a33933bf2b006._comment
new file mode 100644
index 0000000000..42705eede3
--- /dev/null
+++ b/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_2_0a9372f3dba83b6cee3a33933bf2b006._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-14T17:14:51Z"
+ content="""
+Seems that your `.git/annex/othertmp` has somehow became a file.
+git-annex expects it to be a directory, or to not exist, but when it's
+a file, git-annex doesn't understand what is happening and so can fail this
+way.
+
+It's probably safe to delete a `.git/annex/othertmp` file,
+although since you had disk corruption, 
+you might want to take a backup of it first.
+"""]]

diff --git a/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_2_a971d9d85eb7f86d4ea18a71bd0db25d._comment b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_2_a971d9d85eb7f86d4ea18a71bd0db25d._comment
new file mode 100644
index 0000000000..8a1e2aff7d
--- /dev/null
+++ b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_2_a971d9d85eb7f86d4ea18a71bd0db25d._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-14T17:01:37Z"
+ content="""
+The rationelle for this difference is that 
+with `git-annex get` you are explicitly asking it to get
+the specified files (or all files), while with `git-annex sync --content`
+you are asking it to make the local and remote be in sync with respect to
+whether they contain the content or not. When neither local nor remote
+contains the content, they are still in sync.. 
+"""]]

diff --git a/doc/git-annex-sync.mdwn b/doc/git-annex-sync.mdwn
index aac377f4a1..572abec586 100644
--- a/doc/git-annex-sync.mdwn
+++ b/doc/git-annex-sync.mdwn
@@ -175,7 +175,8 @@ have the same value as the currently checked out branch.
 * `--cleanup`
 
   Removes the local and remote `synced/` branches, which were created
-  and pushed by `git-annex sync`.
+  and pushed by `git-annex sync`. This option prevents all other syncing
+  activities.
 
   This can come in handy when you've synced a change to remotes and now
   want to reset your master branch back before that change. So you

diff --git a/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__/comment_1_51e5f78a1cbd37598f3f0f6746df7320._comment b/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__/comment_1_51e5f78a1cbd37598f3f0f6746df7320._comment
new file mode 100644
index 0000000000..e944e69bc3
--- /dev/null
+++ b/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__/comment_1_51e5f78a1cbd37598f3f0f6746df7320._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-14T16:45:31Z"
+ content="""
+`git-annex sync --cleanup` does not do any of the pushing and pulling that
+`git-annex sync` does without the option. So you would want to run the
+command once without the option, and then once with it.
+
+Besides being extra work, `git-annex sync --cleanup` can delete sync
+branches that have not yet been merged. That's the point of it after all;
+the docs for it discuss a situation where you would want it to do that. But
+usually you do want to let the sync branches be merged.
+
+My recommendation would be to not use the option unless you actually need
+it.
+"""]]

diff --git a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment
index ff2d186068..15052ef9d7 100644
--- a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment
+++ b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment
@@ -7,5 +7,7 @@
  content="""
 Oh, nvm, there already seem to be some references in the code to it.
 
-Maybe this should be added under [[https://git-annex.branchable.com/tips/unlocked_files/]]?
+Maybe it should be added that it does that under [[https://git-annex.branchable.com/tips/unlocked_files/]] under using less disk space?
+
+Also, macOS uses a `-c` flag instead of `reflink`, according to [the cp man page](https://www.unix.com/man-page/mojave/1/cp/).
 """]]

diff --git a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment
new file mode 100644
index 0000000000..ff2d186068
--- /dev/null
+++ b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files/comment_1_52db92711e28abb0eb7a346646b91573._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="lena.wildervanck@4b6aac156870f72a36b090e210e4747f702b69cb"
+ nickname="lena.wildervanck"
+ avatar="http://cdn.libravatar.org/avatar/8a423528476e6f6dc6a06588b5bbf457"
+ subject="comment 1"
+ date="2022-11-13T19:03:48Z"
+ content="""
+Oh, nvm, there already seem to be some references in the code to it.
+
+Maybe this should be added under [[https://git-annex.branchable.com/tips/unlocked_files/]]?
+"""]]

diff --git a/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn
new file mode 100644
index 0000000000..72b8bf7330
--- /dev/null
+++ b/doc/todo/Copy-on-Write__47__reflink_support_for_unlocked_files.mdwn
@@ -0,0 +1 @@
+Some filesystems, like BTRFS or ZFS have copy on write support, which means that if files are copied in a special way they share sectors until one of the files is edited. Implementing support would make files take up less space if they are in unlocked mode and not significantly edited.

diff --git a/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_1_41d3a4774dd5ada5036247eb6f562392._comment b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_1_41d3a4774dd5ada5036247eb6f562392._comment
new file mode 100644
index 0000000000..4e72320fa7
--- /dev/null
+++ b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__/comment_1_41d3a4774dd5ada5036247eb6f562392._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="pat"
+ avatar="http://cdn.libravatar.org/avatar/6b552550673a6a6df3b33364076f8ea8"
+ subject="comment 1"
+ date="2022-11-12T02:50:07Z"
+ content="""
+I guess I can `git annex sync --content && git annex get --auto` to make sure I get the content I want. Although at that point, it's probably better to do `git annex sync && git annex get --auto && git annex drop --auto`. I was expecting `sync` to handle the auto get/drop - which it usually does. But it silently passes even when it doesn't get the wanted content.
+
+Perhaps I should think of `sync` as a \"best effort\" mode? That is, it will download available content, but I shouldn't rely on it to download all wanted content - `get` is the job for that.
+"""]]

diff --git a/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__.mdwn b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__.mdwn
new file mode 100644
index 0000000000..c9abafa6e8
--- /dev/null
+++ b/doc/forum/__96__sync_--content__96___error_if_it_can__39__t_get_wanted__63__.mdwn
@@ -0,0 +1,14 @@
+I find that it's possible for a repo to not get the content that it wants to have. It happens if the repo only points to remotes that don't have the content.
+
+`git annex get` will fail:
+
+    $ git annex get --want-get
+    get foo (not available) 
+      Maybe add some of these git remotes (git remote add ...):
+      	6ac12bd5-b585-4884-b0fe-a48fdc1b6365 -- b
+    failed
+    get: 1 failed
+
+But `git annex sync --content` passes, despite not getting any of the wanted content.
+
+Is there a way to make `git annex sync --content` error if it can't fetch wanted content, the way `git annex get` does?

diff --git a/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__.mdwn b/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__.mdwn
new file mode 100644
index 0000000000..8b3fb05a84
--- /dev/null
+++ b/doc/forum/Is___96__sync_--cleanup__96___by_default_a_bad_idea__63__.mdwn
@@ -0,0 +1,3 @@
+While going through the docs, I found that `sync --cleanup` will remove the `synced/*` branches after completing a sync.
+
+Is there any reason why it would be a bad idea to use that flag as a default for my own use. I like the clean aspect of not having those branches around, but I don't know if there's a good reason why I would want to keep them.

diff --git a/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_1_6fb34a361417e4cac08edc20fd2bc3e7._comment b/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_1_6fb34a361417e4cac08edc20fd2bc3e7._comment
new file mode 100644
index 0000000000..3189625f3f
--- /dev/null
+++ b/doc/forum/othertmp__58___createDirectory__58___already_exists/comment_1_6fb34a361417e4cac08edc20fd2bc3e7._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="amindfv@97236fbaab6048ce6805b2737b27dd7f1cd51da4"
+ nickname="amindfv"
+ avatar="http://cdn.libravatar.org/avatar/a82f30bc12d79fee2d142021af8e9aa0"
+ subject="comment 1"
+ date="2022-11-10T21:29:33Z"
+ content="""
+If it matters for the answer, the data I was in the middle of `git annex add`-ing is replaceable. I.e. I'd be perfectly happy to bring my repo back to the state before the `add` and simply copy the data over again and then `add` it again.
+
+Also my version is 8.20210223
+
+"""]]

diff --git a/doc/forum/othertmp__58___createDirectory__58___already_exists.mdwn b/doc/forum/othertmp__58___createDirectory__58___already_exists.mdwn
new file mode 100644
index 0000000000..fb227559d8
--- /dev/null
+++ b/doc/forum/othertmp__58___createDirectory__58___already_exists.mdwn
@@ -0,0 +1,38 @@
+Through no fault of Git Annex's, my drive lost power in the middle of a `git annex add`, and seems to have experienced corruption.
+
+(If it matters, here's the log of what was happening when the drive powered off:)
+
+```
+# [...snip...]
+add disk/DCIM/101_FUJI/DSCF3929.MOV
+ok
+add disk/DCIM/101_FUJI/DSCF3930.MOV
+ok
+add disk/DCIM/101_FUJI/DSCF3931.MOV
+0%    31.98 KiB        67 MiB/s 5s
+git-annex: ../.git/annex/othertmp/ingest-DSCF3931423025-19.MOV: hGetBuf: hardware fault (Input/output error)
+failed
+fatal: not a git repository: '../.git'
+fatal: unable to access '../.git/config': Input/output error
+git-annex: ../.git/annex: createDirectory: hardware fault (Input/output error)
+```
+
+
+Right after it happened, I tried to check on the data and got this error:
+
+```
+$ git annex fsck                                                    
+fatal: /media/foo/bar/.git/annex/index: index file open failed: Structure needs cleaning                                                                                                   
+git-annex: fd:13: hPutBuf: resource vanished (Broken pipe)
+```
+
+I fixed this (I hope) with an `fsck` of the drive (not a `git fsck` or `git annex fsck` - an actual `fsck` to repair the filesystem). Now I want to check that everything's okay with the data, and I run:
+
+```
+$ git annex fsck
+git-annex: .git/annex/othertmp: createDirectory: already exists (File exists)
+```
+
+How can I get past this error?
+
+Thanks!!

diff --git a/CHANGELOG b/CHANGELOG
index ed74058f8a..6be0eeb034 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,11 +1,13 @@
 git-annex (10.20221104) UNRELEASED; urgency=medium
 
+  * export: Fix a bug that left a file on a special remote when
+    two files with the same content were both deleted in the exported tree.
   * S3: Support signature=anonymous to access a S3 bucket anonymously.
     This can be used, for example, with importtree=yes to import from
     a public bucket.
     This feature needs git-annex to be built with aws-0.23.
   * stack.yaml: Updated to lts-19.32 
- 
+
  -- Joey Hess <id@joeyh.name>  Thu, 03 Nov 2022 14:07:31 -0400
 
 git-annex (10.20221103) upstream; urgency=medium
diff --git a/Command/Export.hs b/Command/Export.hs
index d331bb3030..7a08cf68ff 100644
--- a/Command/Export.hs
+++ b/Command/Export.hs
@@ -149,23 +149,24 @@ changeExport r db (ExportFiltered new) = do
 		[] -> updateExportTree db emptyTree new
 		[oldtreesha] -> do
 			diffmap <- mkDiffMap oldtreesha new db
-			let seekdiffmap a = commandActions $ 
-				map a (M.toList diffmap)
+			let seekdiffmap a = mapM_ a (M.toList diffmap)
 			-- Rename old files to temp, or delete.
-			seekdiffmap $ \(ek, (moldf, mnewf)) -> do
-				case (moldf, mnewf) of
-					(Just oldf, Just _newf) ->
+			let deleteoldf = \ek oldf -> commandAction $
+				startUnexport' r db oldf ek
+			seekdiffmap $ \case
+				(ek, (oldf:oldfs, _newf:_)) -> do
+					commandAction $
 						startMoveToTempName r db oldf ek
-					(Just oldf, Nothing) ->
-						startUnexport' r db oldf ek
-					_ -> stop
+					forM_ oldfs (deleteoldf ek)
+				(ek, (oldfs, [])) ->
+					forM_ oldfs (deleteoldf ek)
+				(_ek, ([], _)) -> noop
 			waitForAllRunningCommandActions
 			-- Rename from temp to new files.
-			seekdiffmap $ \(ek, (moldf, mnewf)) ->
-				case (moldf, mnewf) of
-					(Just _oldf, Just newf) ->
-						startMoveFromTempName r db ek newf
-					_ -> stop
+			seekdiffmap $ \case
+				(ek, (_oldf:_, newf:_)) -> commandAction $
+					startMoveFromTempName r db ek newf
+				_ -> noop
 			waitForAllRunningCommandActions
 		ts -> do
 			warning "Resolving export conflict.."
@@ -204,7 +205,7 @@ changeExport r db (ExportFiltered new) = do
 		void $ liftIO cleanup
 
 -- Map of old and new filenames for each changed Key in a diff.
-type DiffMap = M.Map Key (Maybe TopFilePath, Maybe TopFilePath)
+type DiffMap = M.Map Key ([TopFilePath], [TopFilePath])
 
 mkDiffMap :: Git.Ref -> Git.Ref -> ExportHandle -> Annex DiffMap
 mkDiffMap old new db = do
@@ -213,14 +214,14 @@ mkDiffMap old new db = do
 	void $ liftIO cleanup
 	return diffmap
   where
-	combinedm (srca, dsta) (srcb, dstb) = (srca <|> srcb, dsta <|> dstb)
+	combinedm (srca, dsta) (srcb, dstb) = (srca ++ srcb, dsta ++ dstb)
 	mkdm i = do
 		srcek <- getek (Git.DiffTree.srcsha i)
 		dstek <- getek (Git.DiffTree.dstsha i)
 		updateExportTree' db srcek dstek i
 		return $ catMaybes
-			[ (, (Just (Git.DiffTree.file i), Nothing)) <$> srcek
-			, (, (Nothing, Just (Git.DiffTree.file i))) <$> dstek
+			[ (, ([Git.DiffTree.file i], [])) <$> srcek
+			, (, ([], [Git.DiffTree.file i])) <$> dstek
 			]
 	getek sha
 		| sha `elem` nullShas = return Nothing
diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
index d06fe59a96..f73238565a 100644
--- a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
@@ -395,3 +395,5 @@ ok
 Thank you for all your work making this wonderful tool!
 
 [[!meta title="export tree bug when two files with the same content both should be removed"]]
+
+> [[fixed|done]] --[[Joey]]

diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_4_a6490af0427bbe4363bea824d55a7593._comment b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_4_a6490af0427bbe4363bea824d55a7593._comment
new file mode 100644
index 0000000000..18cb92cde5
--- /dev/null
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_4_a6490af0427bbe4363bea824d55a7593._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="nell@7201fe78ade251118ef3441f4e509b37cd836503"
+ nickname="nell"
+ avatar="http://cdn.libravatar.org/avatar/7d9b55f7fec5eaba6fcd3fecc65ccb7d"
+ subject="comment 4"
+ date="2022-11-09T19:51:20Z"
+ content="""
+Nice, thank you for looking into this, Joey! That makes sense why it would appear to be git files, we tend to have identical duplicate metadata files much more often than duplicate image files within datasets.
+"""]]

diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
index ad68504451..d06fe59a96 100644
--- a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
@@ -394,4 +394,4 @@ ok
 
 Thank you for all your work making this wonderful tool!
 
-[[!meta title="export tree sometimes does not delete removed git files"]]
+[[!meta title="export tree bug when two files with the same content both should be removed"]]
diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_3_176cbc137afb5cf8841ff9114b111fef._comment b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_3_176cbc137afb5cf8841ff9114b111fef._comment
new file mode 100644
index 0000000000..0d8f4ce9e4
--- /dev/null
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_3_176cbc137afb5cf8841ff9114b111fef._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2022-11-09T19:41:33Z"
+ content="""
+Ok, the bug is due to 2 files that have the same content.
+
+	sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	sub-000101/ses-displaced/pet/sub-000101_ses-displaced_rec-MLEM_pet.json
+
+Both files get deleted. And the bug makes it only pick one of the two files
+to delete, because it's using a map from key to file and the second file
+overwrites the first in the map.
+
+So this would also presumably affect annexed files when two have the same 
+content and are being deleted.
+"""]]

diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
index 930f80a205..ad68504451 100644
--- a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
@@ -393,3 +393,5 @@ ok
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Thank you for all your work making this wonderful tool!
+
+[[!meta title="export tree sometimes does not delete removed git files"]]
diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_2_059b9beb31d9cbc97ea4a59f47d2e63d._comment b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_2_059b9beb31d9cbc97ea4a59f47d2e63d._comment
new file mode 100644
index 0000000000..dae5c72f0c
--- /dev/null
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_2_059b9beb31d9cbc97ea4a59f47d2e63d._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-09T19:15:59Z"
+ content="""
+Interestingly, it does not happen in simpler situations:
+
+	$ git checkout 1.0.0
+	$ git-annex export 1.0.0 --to d
+	$ git rm sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	$ git commit -m removed
+	$ git-annex export HEAD --to d
+	unexport d sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json ok
+	$ ls ../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	ls: cannot access '../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json': No such file or directory
+
+So something about the diff between 1.0.0 and 1.0.1 is somehow causing the
+bug..
+"""]]

diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_1_70428ebc4027538253edc483dc5cb971._comment b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_1_70428ebc4027538253edc483dc5cb971._comment
new file mode 100644
index 0000000000..9a016f3498
--- /dev/null
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects/comment_1_70428ebc4027538253edc483dc5cb971._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-09T18:16:29Z"
+ content="""
+I tried making a repository with just 2 files, one in git and one in git-annex,
+and am unable to reproduce the bug. Here is 
+what `git-annex export master --to remote --debug` showed when
+exporting a tree that deleted file "foo" which was a git object:
+
+	[2022-11-09 14:16:09.513485291] (Remote.S3) String to sign: "DELETE\n/foo\n\nhost:t-a9b9d406-30e5-41cc-a74c-c5d83b2953fb.s3.amazonaws.com\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:20221109T181609Z\n\nhost;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	[2022-11-09 14:16:09.513608493] (Remote.S3) Host: "t-a9b9d406-30e5-41cc-a74c-c5d83b2953fb.s3.amazonaws.com"
+	[2022-11-09 14:16:09.513697445] (Remote.S3) Path: "/foo"
+	[2022-11-09 14:16:09.513748086] (Remote.S3) Query string: ""
+	[2022-11-09 14:16:09.513829584] (Remote.S3) Header: (redacted -- JEH)
+	[2022-11-09 14:16:09.687814925] (Remote.S3) Response status: Status {statusCode = 204, statusMessage = "No Content"}
+
+The S3 console showed that the file was deleted from the bucket.
+And as far as the S3 remote implementation is concerned, there should
+not be anything different between a git object and a git-annex object.
+At the level of the S3 remote both have a git-annex key that it deletes
+in the same way.
+
+In your log, the only thing it does with the file is export it, but it does
+not later unexport it:
+
+	$ grep baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json Versioned_S3_tree_does_not_unexport_git_objects.mdwn
+	Git files in the 1.0.0 tag are still present in the S3 1.0.1 export. sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json is an example file not present in 1.0.1 that is still present on S3.
+	export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json 
+	$
+
+If the bug is that it's somehow failing to try to unexport the file,
+that should happen independently of the special remote type, so would also
+happen with a directory special remote. So I tried that:
+
+	$ git clone https://github.com/openneurodatasets/ds001705
+	$ cd ds001705
+	$ git-annex get --branch=tags/1.0.0
+	$ git-annex get --branch=tags/1.0.1
+	$ mkdir ../d
+	$ git-annex initremote d type=directory directory=../d encryption=none exporttree=yes
+ 	$ git-annex export 1.0.0 --to d
+	$ ls ../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	$ git-annex export 1.0.1 --to d
+	$ ls ../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+	../d/sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json
+
+Ok, so, nothing to do with S3 or versioning at all.
+"""]]

diff --git a/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn b/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn
index de709532a0..6f31c3303e 100644
--- a/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn
+++ b/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn
@@ -49,3 +49,5 @@ Sure, I use it several times a week with my multigigabyte backups, where it give
 
 [[!meta author=jkniiv]]
 [[!meta title="introduction of aws-0.23 causes stack build to fail"]]
+
+> [[fixed|done]] --[[Joey]]

diff --git a/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
new file mode 100644
index 0000000000..930f80a205
--- /dev/null
+++ b/doc/bugs/Versioned_S3_tree_does_not_unexport_git_objects.mdwn
@@ -0,0 +1,395 @@
+### Please describe the problem.
+
+When exporting commits to S3 with versioning=yes and exporttree=yes, deleted annexed objects are marked as deleted but deleted git objects remain. This prevents a user from retrieving a consistent version of the exporttree commit without using git-annex.
+
+### What steps will reproduce the problem?
+
+* Create a repository and add annexed and git objects.
+* Add S3 special remote with exporttree=yes and versioning=yes
+* Export to the remote.
+* Remove a git object and an annexed object.
+* Export again and the annexed object will be removed while the git object remains.
+
+Example repository and publicly readable remote having this issue here:
+
+https://github.com/openneurodatasets/ds001705
+
+Git files in the 1.0.0 tag are still present in the S3 1.0.1 export. sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json is an example file not present in 1.0.1 that is still present on S3.
+
+### What version of git-annex are you using? On what operating system?
+
+Debian bullseye
+
+```
+git-annex version: 10.20220927-ga92546587
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+```
+
+### Please provide any additional information below.
+
+[[!format sh """
+# git annex export 1.0.0 --to=s3-PUBLIC
+export s3-PUBLIC .bidsignore 
+ok                                
+export s3-PUBLIC .datalad/.gitattributes 
+ok                                
+export s3-PUBLIC .datalad/config 
+ok                                
+export s3-PUBLIC .gitattributes 
+ok                                
+export s3-PUBLIC CHANGES 
+ok                                
+export s3-PUBLIC README 
+ok                                
+export s3-PUBLIC dataset_description.json 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/anat/sub-000101_ses-baseline_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/anat/sub-000101_ses-baseline_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_K1.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_Vb.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_k2.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_k4.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000101/ses-baseline/pet/sub-000101_ses-baseline_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/anat/sub-000101_ses-displaced_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/anat/sub-000101_ses-displaced_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/pet/sub-000101_ses-displaced_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/pet/sub-000101_ses-displaced_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000101/ses-displaced/pet/sub-000101_ses-displaced_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/anat/sub-000102_ses-baseline_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/anat/sub-000102_ses-baseline_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_K1.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_Vb.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_k2.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_k4.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000102/ses-baseline/pet/sub-000102_ses-baseline_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/anat/sub-000102_ses-displaced_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/anat/sub-000102_ses-displaced_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/pet/sub-000102_ses-displaced_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/pet/sub-000102_ses-displaced_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000102/ses-displaced/pet/sub-000102_ses-displaced_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/anat/sub-000103_ses-baseline_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/anat/sub-000103_ses-baseline_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_K1.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_Vb.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_k2.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_k4.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000103/ses-baseline/pet/sub-000103_ses-baseline_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/anat/sub-000103_ses-displaced_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/anat/sub-000103_ses-displaced_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/pet/sub-000103_ses-displaced_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/pet/sub-000103_ses-displaced_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000103/ses-displaced/pet/sub-000103_ses-displaced_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/anat/sub-000104_ses-baseline_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/anat/sub-000104_ses-baseline_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_K1.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_Vb.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_k2.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_k4.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000104/ses-baseline/pet/sub-000104_ses-baseline_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/anat/sub-000104_ses-displaced_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/anat/sub-000104_ses-displaced_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/pet/sub-000104_ses-displaced_k3.nii.gz 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/pet/sub-000104_ses-displaced_rec-MLEM_pet.json 
+ok                                
+export s3-PUBLIC sub-000104/ses-displaced/pet/sub-000104_ses-displaced_rec-MLEM_pet.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/anat/sub-000105_ses-baseline_T1w.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/anat/sub-000105_ses-baseline_label-displacementROI_dseg.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/pet/.bidsignore 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/pet/sub-000105_ses-baseline_K1.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/pet/sub-000105_ses-baseline_Vb.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/pet/sub-000105_ses-baseline_k2.nii.gz 
+ok                                
+export s3-PUBLIC sub-000105/ses-baseline/pet/sub-000105_ses-baseline_k3.nii.gz 
+ok                                

(Diff truncated)

diff --git a/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn b/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn
new file mode 100644
index 0000000000..de709532a0
--- /dev/null
+++ b/doc/bugs/introduction_of_aws-0.23_causes_stack_bld_to_fail.mdwn
@@ -0,0 +1,51 @@
+### Please describe the problem.
+
+After aws-0.23 was introduced into extra-deps of `stack.yaml`, a stack build fails
+in the dependency resolution phase (a plan construction problem). Adding `aeson-2.1.1.0`
+into extra-deps as advised leads to a dependency hell with conflicting versions of
+attoparsec being required amongst other troubles.
+
+### What steps will reproduce the problem?
+
+`stack setup && stack build`. Observe the following output:
+
+[[!format sh """
+Error: While constructing the build plan, the following exceptions were encountered:
+
+In the dependencies for aws-0.23:
+    aeson-1.5.6.0 from stack configuration does not match >=2.0.0.0  (latest matching version
+                  is 2.1.1.0)
+needed due to git-annex-10.20221103 -> aws-0.23
+
+Some different approaches to resolving this:
+
+  * Set 'allow-newer: true'
+    in C:\hs-stack\config.yaml to ignore all version constraints and build anyway.
+
+  * Recommended action: try adding the following to your extra-deps
+    in C:\Projektit\git-annex.branchable.com\git-annex--BUILD-221108-4f6c6114f\stack.yaml:
+
+- aeson-2.1.1.0@sha256:103ceb1421cd0ffa810bfb1acb1261d60addbde1a041fb5cce0056ff7d7dcdc2,5980
+
+Plan construction failed.
+# End of transcript or log.
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+I was trying to build master or [[!commit 4f6c6114fb487b5f84aa7e6922786b0e19806525]], but alas no luck.
+
+Windows 10 version 22H2 (build 19045.2130), 64 bit.
+
+Stack version 2.9.1, Git revision 409d56031b4240221d656db09b2ba476fe6bb5b1 x86_64 hpack-0.35.0 .
+
+### Please provide any additional information below.
+
+.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Sure, I use it several times a week with my multigigabyte backups, where it gives structure to my image-based backup routines, so you could say I'm a believer. :)
+
+[[!meta author=jkniiv]]
+[[!meta title="introduction of aws-0.23 causes stack build to fail"]]

diff --git a/Test.hs b/Test.hs
index cb0c515ee9..6d3c58d3c5 100644
--- a/Test.hs
+++ b/Test.hs
@@ -355,7 +355,15 @@ repoTests note numparts = map mk $ sep
 	mk l = testGroup groupname (initTests : map adddep l)
 	adddep = Test.Tasty.after AllSucceed (groupname ++ "." ++ initTestsName)
 	groupname = "Repo Tests " ++ note
-	sep = sep' (replicate numparts [])
+	sep l = 
+		-- Avoid separating into parts that contain less than
+		-- 5 tests each. Since the tests depend on the
+		-- initTests, this avoids spending too much work running
+		-- the initTests once per part.
+		let numparts' = if length l `div` numparts > 5
+			then numparts
+			else length l `div` 5
+		in sep' (replicate numparts' []) l
 	sep' (p:ps) (l:ls) = sep' (ps++[l:p]) ls
 	sep' ps [] = ps
 	sep' [] _ = []
diff --git a/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment
new file mode 100644
index 0000000000..2118ae0519
--- /dev/null
+++ b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_2_e7da4a54b27f459d528a7651fd615913._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-11-07T17:07:52Z"
+ content="""
+I wondered if running the add and init tests once per worker was causing
+a diminishing returns at higher -J levels. Since at -J16, each worker runs
+the add and init tests followed by only 2 or 3 other tests, the time spent
+on the add and init tests becomes more and more significant.
+
+Timings from my laptop (with 4 cores):
+
+	-J16    250 seconds
+	-J8     212 seconds
+	-J4     214 seconds
+        -J2     307 seconds
+
+So a small diminishing returns at -J16.
+
+After some improvements:
+
+	-J16    223 seconds
+	-J8     218 seconds
+	-J4     214 seconds
+
+That won't solve smaug being so overloaded though.
+If anything, it will just make it get to some other test before it times out..
+"""]]

diff --git a/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_1_ab2c1db51a42a146708b28f4cb3309a6._comment b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_1_ab2c1db51a42a146708b28f4cb3309a6._comment
new file mode 100644
index 0000000000..ebda9a160d
--- /dev/null
+++ b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion/comment_1_ab2c1db51a42a146708b28f4cb3309a6._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-07T16:32:15Z"
+ content="""
+I notice that just the init test is taking 346 seconds to run. Then the add
+test is taking 499 seconds. Those are both tests that do very little work,
+and here run in 1s or so. The only thing "wrong" with the conflict resolution tests
+may be that they do 10x as much work.
+
+In the same log, the quickcheck tests, which are entirely CPU bound, ran at normal
+speed. So probably it's disk IO slowing the other tests down.
+
+Also, `result-smaug-900/handle-result.yaml-437-402e214e-success/result-smaug-900/git-annex.log`
+times out on another test. So I'm sure the problem is not a specific test stalling.
+
+And in `result-smaug-899/handle-result.yaml-435-3e89687a-success/result-smaug-899/git-annex.log`,
+it succeeds in 856 seconds. That still seems slow, my laptop can run it in 250 seconds
+with 4 cores (even when using -J16). That makes me think that the IO load (or whatever)
+on smaug is varying and slowing down the test by different amounts.
+
+Smaug has 16 cores. The test suite automatically parallelizes, so there
+will be 16 worker processes. But each worker process has to itself 
+run the init and add tests, which are prereqs for most other tests. So those
+tests are being run 16 times. Which just by itself, at the 346 and 499 second
+run times, will take more than an hour. Also if disk IO is starved, a lot
+of workers is only going to thrash it more.
+
+So `git-annex test -J1` or so seems like a good idea.
+
+It might also be better to not have a single timeout for the whole
+test suite, but time out if it stalls for long enough without generating new
+output.
+"""]]

diff --git a/doc/bugs/chunks_not_deduplicated.mdwn b/doc/bugs/chunks_not_deduplicated.mdwn
index 80bf65465c..9a70aae973 100644
--- a/doc/bugs/chunks_not_deduplicated.mdwn
+++ b/doc/bugs/chunks_not_deduplicated.mdwn
@@ -23,3 +23,5 @@ I imagine an optional upgrade to a new key format for chunks, such that they had
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yesterday I recovered a bricked phone by accessing flash partitions I had backed up with git-annex. I deleted them locally by accident, by git-annex cobbled many of them together from remotes. The ones it didn't find still had their hashes preserved in the repository, and I was able to compare with hashes on the device and reconstructed all but one. The one I didn't reconstruct is regenerateable.
+
+> [[notabug|done]] --[[Joey]]
diff --git a/doc/bugs/chunks_not_deduplicated/comment_1_3890af3669ac6b37b88dac938ed79d16._comment b/doc/bugs/chunks_not_deduplicated/comment_1_3890af3669ac6b37b88dac938ed79d16._comment
new file mode 100644
index 0000000000..036afc770a
--- /dev/null
+++ b/doc/bugs/chunks_not_deduplicated/comment_1_3890af3669ac6b37b88dac938ed79d16._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-11-07T16:25:27Z"
+ content="""
+See [[todo/option_to_individually_hash_chunks]].
+"""]]

diff --git a/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion.mdwn b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion.mdwn
new file mode 100644
index 0000000000..075a1841ce
--- /dev/null
+++ b/doc/bugs/conflict_resolution___40____63____41___test_stalls_on_an_occasion.mdwn
@@ -0,0 +1,40 @@
+### Please describe the problem.
+
+In a daily testing of git-annex on client systems, on `smaug` we observed tests stalling and being eventually killed once in a while consistently at `conflict resolution (mixed locked and unlocked file):` step:
+
+```
+(git)smaug:/mnt/datasets/datalad/ci/git-annex-ci-client-jobs[master]builds/2022
+$> datalad foreach-dataset --o-s relpath git grep 'Elapsed time: 3600 seconds'
+06/result-smaug-720/handle-result.yaml-164-4355efb7-success/result-smaug-720/git-annex.log:    conflict resolution (nonannexed symlink): + echo 'Elapsed time: 3600 seconds'                                                                                       
+06/result-smaug-720/handle-result.yaml-164-4355efb7-success/result-smaug-720/git-annex.log:Elapsed time: 3600 seconds
+11/result-smaug-900/handle-result.yaml-437-402e214e-success/result-smaug-900/git-annex.log:        present False:                                   + echo 'Elapsed time: 3600 seconds'                                                                            
+11/result-smaug-900/handle-result.yaml-437-402e214e-success/result-smaug-900/git-annex.log:Elapsed time: 3600 seconds
+08/result-smaug-787/handle-result.yaml-276-ca542974-success/result-smaug-787/git-annex.log:    conflict resolution (nonannexed symlink): + echo 'Elapsed time: 3600 seconds'                                                                                       
+08/result-smaug-787/handle-result.yaml-276-ca542974-success/result-smaug-787/git-annex.log:Elapsed time: 3600 seconds
+10/result-smaug-853/handle-result.yaml-373-7e7a9411-success/result-smaug-853/git-annex.log:    conflict resolution (mixed locked and unlocked file): + echo 'Elapsed time: 3600 seconds'                                                                           
+10/result-smaug-853/handle-result.yaml-373-7e7a9411-success/result-smaug-853/git-annex.log:Elapsed time: 3600 seconds
+10/result-smaug-854/handle-result.yaml-375-d95326d3-success/result-smaug-854/git-annex.log:    conflict resolution (nonannexed symlink): + echo 'Elapsed time: 3600 seconds'
+10/result-smaug-854/handle-result.yaml-375-d95326d3-success/result-smaug-854/git-annex.log:Elapsed time: 3600 seconds
+```
+
+so far seemed to happen only on smaug among "test clients". 
+
+### What version of git-annex are you using? On what operating system?
+
+in above `grep` the versions are:
+
+```
+(git)smaug:/mnt/datasets/datalad/ci/git-annex-ci-client-jobs[master]builds/2022
+$> datalad foreach-dataset --o-s relpath git grep -l 'Elapsed time: 3600 seconds' | xargs grep '^git-annex version:' | sort -n
+06/result-smaug-720/handle-result.yaml-164-4355efb7-success/result-smaug-720/git-annex.log:git-annex version: 10.20220525+git47-g4bf796225-1~ndall+1
+08/result-smaug-787/handle-result.yaml-276-ca542974-success/result-smaug-787/git-annex.log:git-annex version: 10.20220724+git77-ga24ae0814-1~ndall+1
+10/result-smaug-853/handle-result.yaml-373-7e7a9411-success/result-smaug-853/git-annex.log:git-annex version: 10.20220927+git27-g82dab0749-1~ndall+1
+10/result-smaug-854/handle-result.yaml-375-d95326d3-success/result-smaug-854/git-annex.log:git-annex version: 10.20220927+git27-g82dab0749-1~ndall+1
+11/result-smaug-900/handle-result.yaml-437-402e214e-success/result-smaug-900/git-annex.log:git-annex version: 10.20221103+git9-gad7dfdb37-1~ndall+1
+```
+
+full logs are available on smaug at the paths listed above
+
+[[!meta author=yoh]]
+[[!tag projects/datalad]]
+

diff --git a/doc/bugs/chunks_not_deduplicated.mdwn b/doc/bugs/chunks_not_deduplicated.mdwn
new file mode 100644
index 0000000000..80bf65465c
--- /dev/null
+++ b/doc/bugs/chunks_not_deduplicated.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+When data is chunked for storage on a remote, identical chunks may be stored repeatedly with different hashes.
+
+### What steps will reproduce the problem?
+```
+git annex init .
+dd count=16 bs=1024 if=/dev/urandom of=rand1.bin
+dd count=15 skip=1 bs=1024 if=rand1.bin of=rand2.bin
+git annex add rand1.bin rand2.bin
+mkdir data
+git annex initremote data type=directory directory=data encryption=none chunk=1024
+git annex copy --all --to=data
+find data -type f | xargs sha1sum | sort # shows doubly-stored identical chunks
+```
+
+### What version of git-annex are you using? On what operating system?
+10.20221004-gbf27a02b0 on redhat 7
+
+### Please provide any additional information below.
+
+I imagine an optional upgrade to a new key format for chunks, such that they had their own hashes, and the larger key they were a part of were identified separately (e.g. in data content or metadata). They would then become deduplicated already using existing means.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yesterday I recovered a bricked phone by accessing flash partitions I had backed up with git-annex. I deleted them locally by accident, by git-annex cobbled many of them together from remotes. The ones it didn't find still had their hashes preserved in the repository, and I was able to compare with hashes on the device and reconstructed all but one. The one I didn't reconstruct is regenerateable.

diff --git a/doc/todo/allow_for_annonymous_AWS_S3_access.mdwn b/doc/todo/allow_for_annonymous_AWS_S3_access.mdwn
index 123bc70f6c..1eabf4a618 100644
--- a/doc/todo/allow_for_annonymous_AWS_S3_access.mdwn
+++ b/doc/todo/allow_for_annonymous_AWS_S3_access.mdwn
@@ -16,3 +16,5 @@ in my searches information on "anonymous access to S3" is scarce, but in DataLad
 
 [[!meta author=yoh]]
 [[!tag projects/dandi]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/todo/allow_for_annonymous_AWS_S3_access/comment_9_7fa3b2aafcad92c20f8a3467aa3ae6b9._comment b/doc/todo/allow_for_annonymous_AWS_S3_access/comment_9_7fa3b2aafcad92c20f8a3467aa3ae6b9._comment
new file mode 100644
index 0000000000..a2a3569074
--- /dev/null
+++ b/doc/todo/allow_for_annonymous_AWS_S3_access/comment_9_7fa3b2aafcad92c20f8a3467aa3ae6b9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2022-11-04T19:46:26Z"
+ content="""
+I've finished up the work on this. To use it, you will need a git-annex
+built with aws 0.23. It will take some time before that is available in
+some builds, but when building git-annex with stack, it will use it
+already.
+"""]]

diff --git a/doc/bugs/move.log__58___openFile__58___resource_busy___40__file_is_locked__41__/comment_2_6216c1eb6991cec787333284463fcc9d._comment b/doc/bugs/move.log__58___openFile__58___resource_busy___40__file_is_locked__41__/comment_2_6216c1eb6991cec787333284463fcc9d._comment
new file mode 100644
index 0000000000..b16d04b181
--- /dev/null
+++ b/doc/bugs/move.log__58___openFile__58___resource_busy___40__file_is_locked__41__/comment_2_6216c1eb6991cec787333284463fcc9d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2022-11-04T12:41:47Z"
+ content="""
+ok, did the archaeologic expedition to figure when fixed -- was  fixed in [10.20221003-19-g4a42c6909 AKA 10.20221103~28](https://git.kitenet.net/index.cgi/git-annex.git/commit/?id=4a42c69092a03cce7b31b79b862e59c9842ced77) , brew still (well -- we are just 1 day post release! ;)) has 10.20221003 so in testing git-annex-remote-rclone we keep getting hit but hopefully it would go away soon with update of git-annex in brew.
+"""]]

diff --git a/doc/news/version_10.20220624.mdwn b/doc/news/version_10.20220624.mdwn
deleted file mode 100644
index f2a6ba5522..0000000000
--- a/doc/news/version_10.20220624.mdwn
+++ /dev/null
@@ -1,23 +0,0 @@
-git-annex 10.20220624 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * init: Added --no-autoenable option.
-  * info: Added --autoenable option.
-  * initremote: Improve handling of type=git special remotes.
-    The location value no longer needs to match the url of an existing
-    git remote, and locations not using ssh:// will work now, including
-    both paths and host:/path
-  * Fix retrival of an empty file that is stored in a special remote with
-    chunking enabled.
-    (Fixes a reversion in 8.20201103)
-  * move: Improve resuming a move that succeeded in transferring the
-    content, but where dropping failed due to eg a network problem,
-    in cases where numcopies checks prevented the resumed
-    move from dropping the object from the source repository.
-  * add, fix, lock, rekey: When several files were being processed,
-    replacing an annex symlink of a file that was already processed
-    with a new large file could sometimes cause that large file to be
-    added to git. These races have been fixed.
-  * add: Also fix a similar race that could cause a large file be added
-    to git when a small file was modified or overwritten while it was
-    being added.
-  * add --batch: Fix handling of a file that is skipped due to being
-    gitignored."""]]
\ No newline at end of file
diff --git a/doc/news/version_10.20221103.mdwn b/doc/news/version_10.20221103.mdwn
new file mode 100644
index 0000000000..e7374b7c48
--- /dev/null
+++ b/doc/news/version_10.20221103.mdwn
@@ -0,0 +1,22 @@
+git-annex 10.20221103 released with [[!toggle text="these changes"]]
+[[!toggleable text="""  * Doubled the speed of git-annex drop when operating on many files,
+    and of git-annex get when operating on many tiny files.
+  * trust, untrust, semitrust, dead: Fix behavior when provided with
+    multiple repositories to operate on.
+  * trust, untrust, semitrust, dead: When provided with no parameters,
+    do not operate on a repository that has an empty name.
+  * move: Fix openFile crash with -J
+    (Fixes a reversion in 8.20201103)
+  * S3: Speed up importing from a large bucket when fileprefix= is set,
+    by only asking for files under the prefix.
+  * When importing from versioned remotes, fix tracking of the content
+    of deleted files.
+  * More robust handling of ErrorBusy when writing to sqlite databases.
+  * Avoid hanging when a suspended git-annex process is keeping a sqlite
+    database locked.
+  * Make --batch mode handle unstaged annexed files consistently
+    whether the file is unlocked or not. Note that this changes the
+    behavior of --batch when it is provided with locked files that are
+    in the process of being added to the repository, but have not yet been
+    staged in git.
+  * Make git-annex enable-tor work when using the linux standalone build."""]]
\ No newline at end of file

diff --git a/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex.mdwn b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex.mdwn
index c8faf340c2..8a14b35dc8 100644
--- a/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex.mdwn
+++ b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex.mdwn
@@ -49,9 +49,11 @@ This should display something like:
 
 Once you are sure things went on okay, you can synchronise this with `marcos`:
 
-    git annex sync
+    git annex sync --allow-unrelated-histories
 
-This will push the metadata information to marcos, so it knows which files are available on `angela`. From there on, you can freely get and move files between the two repos!
+This will push the metadata information to marcos, so it knows which files
+are available on `angela`. From there on, you can freely get and move files
+between the two repos!
 
 Importing files from a third directory
 --------------------------------------
@@ -61,7 +63,7 @@ Say that some files on `angela` are actually spread out outside of the `~/mp3` d
     cd ~/mp3
     git annex import ~/music/
 
-(!) Be careful that `~/music` is not a git-annex repository, or this will [[destroy it!|bugs/git annex import destroys a fellow git annex repository]].
+(!) Be careful that `~/music` is not a git-annex repository.
 
 Deleting deleted files
 ----------------------
@@ -73,7 +75,3 @@ It is quite possible some files were removed (or renamed!) on `marcos` but not o
 This will show files that are on `angela` and not on `marcos`. They could be new files that were only added on `angela`, so be careful! A manual analysis is necessary, but let's say you are certain those files are not relevant anymore, you can delete them from `angela`:
 
     git annex drop <file>
-
-If the file is a renamed or modified version from the original, you may need to use `--force`, but be careful! If you delete the wrong file, it will be lost forever!
-
-> (!) Maybe this wouldn't happen with [[direct mode]] and an fsck? --[[anarcat]]
diff --git a/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_2_c6c11a9d5f9f136fa541404cdc49f45c._comment b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_2_c6c11a9d5f9f136fa541404cdc49f45c._comment
new file mode 100644
index 0000000000..dc693fa467
--- /dev/null
+++ b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_2_c6c11a9d5f9f136fa541404cdc49f45c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-10-31T16:09:12Z"
+ content="""
+Indeed, you will need to use `git-annex sync --allow-unrelated-histories`
+now in that situation. I have updated the tip.
+"""]]

diff --git a/doc/install/rpm_standalone/comment_3_3a38ab16ca034025476b4df0a566b4a9._comment b/doc/install/rpm_standalone/comment_3_3a38ab16ca034025476b4df0a566b4a9._comment
new file mode 100644
index 0000000000..0e08bcb0c9
--- /dev/null
+++ b/doc/install/rpm_standalone/comment_3_3a38ab16ca034025476b4df0a566b4a9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="xloem"
+ avatar="http://cdn.libravatar.org/avatar/b8c087f7c5e6a9358748f0727c077f3b"
+ subject="ipfs"
+ date="2022-10-31T13:36:51Z"
+ content="""
+It would be nice if the rpm or repository included auxiliarity scripts such as git-annex-remote-ipfs so that these would get installed/uninstalled/upgraded alongside the main project.
+"""]]

diff --git a/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_1_d94ae69945416e57faa6e6dd5536f66e._comment b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_1_d94ae69945416e57faa6e6dd5536f66e._comment
new file mode 100644
index 0000000000..2f739fe461
--- /dev/null
+++ b/doc/tips/migrating_two_seperate_disconnected_directories_to_git_annex/comment_1_d94ae69945416e57faa6e6dd5536f66e._comment
@@ -0,0 +1,59 @@
+[[!comment format=mdwn
+ username="Stefan"
+ avatar="http://cdn.libravatar.org/avatar/1474db4b030b82320e3bd5e899ef2bad"
+ subject="This guide fails with "fatal: refusing to merge unrelated histories""
+ date="2022-10-29T10:28:18Z"
+ content="""
+This no longer works, here is a MWE to copy-paste (uses /tmp/{A,B}):
+
+```
+mkdir /tmp/A && touch /tmp/A/bigfile
+mkdir /tmp/B && touch /tmp/B/bigfile
+cd /tmp/A
+git init
+git annex init
+git annex add .
+git commit -m \"git annex yay\"
+cd /tmp/B
+git init
+git remote add A /tmp/A
+git fetch A
+git annex info # this should display the two repos
+git annex add .
+git annex whereis
+git annex sync
+```
+
+This fails with
+
+```
+commit 
+[main (root-commit) e9435bf] git-annex in stefan@notebook:/tmp/B
+ 1 file changed, 1 insertion(+)
+ create mode 120000 bigfile
+ok
+pull A 
+
+fatal: refusing to merge unrelated histories
+failed
+push A 
+Enumerating objects: 19, done.
+Counting objects: 100% (19/19), done.
+Delta compression using up to 8 threads
+Compressing objects: 100% (11/11), done.
+Writing objects: 100% (14/14), 1.37 KiB | 1.37 MiB/s, done.
+Total 14 (delta 2), reused 0 (delta 0), pack-reused 0
+To /tmp/A
+ * [new branch]      main -> synced/main
+ * [new branch]      git-annex -> synced/git-annex
+To /tmp/A
+ ! [rejected]        main -> main (non-fast-forward)
+error: failed to push some refs to '/tmp/A'
+hint: Updates were rejected because the tip of your current branch is behind
+hint: its remote counterpart. Integrate the remote changes (e.g.
+hint: 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+ok
+sync: 1 failed
+```
+"""]]

diff --git a/Annex/Path.hs b/Annex/Path.hs
index 11400d32a5..e058db32a8 100644
--- a/Annex/Path.hs
+++ b/Annex/Path.hs
@@ -1,6 +1,6 @@
 {- git-annex program path
  -
- - Copyright 2013-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -11,6 +11,7 @@ module Annex.Path (
 	gitAnnexChildProcess,
 	gitAnnexChildProcessParams,
 	gitAnnexDaemonizeParams,
+	cleanStandaloneEnvironment,
 ) where
 
 import Annex.Common
@@ -19,7 +20,7 @@ import Utility.Env
 import Annex.PidLock
 import qualified Annex
 
-import System.Environment (getExecutablePath, getArgs)
+import System.Environment (getExecutablePath, getArgs, getProgName)
 
 {- A fully qualified path to the currently running git-annex program.
  - 
@@ -29,13 +30,16 @@ import System.Environment (getExecutablePath, getArgs)
  - or searching for the command name in PATH.
  -
  - The standalone build runs git-annex via ld.so, and defeats
- - getExecutablePath. It sets GIT_ANNEX_PROGRAMPATH to the correct path
- - to the wrapper script to use.
+ - getExecutablePath. It sets GIT_ANNEX_DIR to the location of the
+ - standalone build directory, and there are wrapper scripts for git-annex
+ - and git-annex-shell in that directory.
  -}
 programPath :: IO FilePath
-programPath = go =<< getEnv "GIT_ANNEX_PROGRAMPATH"
+programPath = go =<< getEnv "GIT_ANNEX_DIR"
   where
-	go (Just p) = return p
+	go (Just dir) = do
+		name <- getProgName
+		return (dir </> name)
 	go Nothing = do
 		exe <- getExecutablePath
 		p <- if isAbsolute exe
@@ -97,3 +101,25 @@ gitAnnexDaemonizeParams = do
 	-- Get every parameter git-annex was run with.
 	ps <- liftIO getArgs
 	return (map Param ps ++ cps)
+
+{- Returns a cleaned up environment that lacks path and other settings
+ - used to make the standalone builds use their bundled libraries and programs.
+ - Useful when calling programs not included in the standalone builds.
+ -
+ - For a non-standalone build, returns Nothing.
+ -}
+cleanStandaloneEnvironment :: IO (Maybe [(String, String)])
+cleanStandaloneEnvironment = clean <$> getEnvironment
+  where
+	clean environ
+		| null vars = Nothing
+		| otherwise = Just $ catMaybes $ map (restoreorig environ) environ
+	  where
+		vars = words $ fromMaybe "" $
+			lookup "GIT_ANNEX_STANDLONE_ENV" environ
+		restoreorig oldenviron p@(k, _v)
+			| k `elem` vars = case lookup ("ORIG_" ++ k) oldenviron of
+				(Just v')
+					| not (null v') -> Just (k, v')
+				_ -> Nothing
+			| otherwise = Just p
diff --git a/Assistant/Install.hs b/Assistant/Install.hs
index 3569ddb4ba..6a31968d7b 100644
--- a/Assistant/Install.hs
+++ b/Assistant/Install.hs
@@ -171,25 +171,3 @@ installFileManagerHooks program = unlessM osAndroid $ do
 #else
 installFileManagerHooks _ = noop
 #endif
-
-{- Returns a cleaned up environment that lacks settings used to make the
- - standalone builds use their bundled libraries and programs.
- - Useful when calling programs not included in the standalone builds.
- -
- - For a non-standalone build, returns Nothing.
- -}
-cleanEnvironment :: IO (Maybe [(String, String)])
-cleanEnvironment = clean <$> getEnvironment
-  where
-	clean environ
-		| null vars = Nothing
-		| otherwise = Just $ catMaybes $ map (restoreorig environ) environ
-	  where
-		vars = words $ fromMaybe "" $
-			lookup "GIT_ANNEX_STANDLONE_ENV" environ
-		restoreorig oldenviron p@(k, _v)
-			| k `elem` vars = case lookup ("ORIG_" ++ k) oldenviron of
-				(Just v')
-					| not (null v') -> Just (k, v')
-				_ -> Nothing
-			| otherwise = Just p
diff --git a/Build/LinuxMkLibs.hs b/Build/LinuxMkLibs.hs
index 4c0824fb79..7beab60125 100644
--- a/Build/LinuxMkLibs.hs
+++ b/Build/LinuxMkLibs.hs
@@ -164,8 +164,6 @@ installLinkerShim top linker exe = do
 		createSymbolicLink (fromRawFilePath link) (top </> exelink)
 	writeFile exe $ unlines
 		[ "#!/bin/sh"
-		, "GIT_ANNEX_PROGRAMPATH=\"$0\""
-		, "export GIT_ANNEX_PROGRAMPATH"
 		, "exec \"$GIT_ANNEX_DIR/" ++ exelink ++ "\" --library-path \"$GIT_ANNEX_LD_LIBRARY_PATH\" \"$GIT_ANNEX_DIR/shimmed/" ++ base ++ "/" ++ base ++ "\" \"$@\""
 		]
 	modifyFileMode (toRawFilePath exe) $ addModes executeModes
diff --git a/CHANGELOG b/CHANGELOG
index 9926f1ff84..42e68dfda7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -17,6 +17,7 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
     database locked.
   * Make --batch mode handle unstaged annexed files consistently
     whether the file is unlocked or not.
+  * Make git-annex enable-tor work when using the linux standalone build.
 
  -- Joey Hess <id@joeyh.name>  Mon, 03 Oct 2022 13:36:42 -0400
 
diff --git a/Command/EnableTor.hs b/Command/EnableTor.hs
index aeae96be92..df518c2fa0 100644
--- a/Command/EnableTor.hs
+++ b/Command/EnableTor.hs
@@ -60,9 +60,10 @@ start _os = do
 			gitannex <- liftIO programPath
 			let ps = [Param (cmdname cmd), Param (show curruserid)]
 			sucommand <- liftIO $ mkSuCommand gitannex ps
+			cleanenv <- liftIO $ cleanStandaloneEnvironment
 			maybe noop showLongNote
 				(describePasswordPrompt' sucommand)
-			ifM (liftIO $ runSuCommand sucommand)
+			ifM (liftIO $ runSuCommand sucommand cleanenv)
 				( next checkHiddenService
 				, giveup $ unwords $
 					[ "Failed to run as root:" , gitannex ] ++ toCommand ps
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index 236a94dac4..1e01e1a97f 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -22,6 +22,7 @@ import Utility.WebApp
 import Utility.Daemon (checkDaemon)
 import Utility.UserInfo
 import Annex.Init
+import Annex.Path
 import qualified Git
 import Git.Types (fromConfigValue)
 import qualified Git.Config
@@ -222,7 +223,7 @@ openBrowser' mcmd htmlshim realurl outh errh =
 #endif
 		hPutStrLn (fromMaybe stdout outh) $ "Launching web browser on " ++ url
 		hFlush stdout
-		environ <- cleanEnvironment
+		environ <- cleanStandaloneEnvironment
 		let p' = p
 			{ env = environ
 			, std_out = maybe Inherit UseHandle outh
diff --git a/Utility/Su.hs b/Utility/Su.hs
index 52f3f7f687..e956d808b4 100644
--- a/Utility/Su.hs
+++ b/Utility/Su.hs
@@ -57,9 +57,9 @@ describePasswordPrompt' :: Maybe SuCommand -> Maybe String
 describePasswordPrompt' (Just (SuCommand p _ _)) = describePasswordPrompt p
 describePasswordPrompt' Nothing = Nothing
 
-runSuCommand :: (Maybe SuCommand) -> IO Bool
-runSuCommand (Just (SuCommand _ cmd ps)) = boolSystem cmd ps
-runSuCommand Nothing = return False
+runSuCommand :: (Maybe SuCommand) -> Maybe [(String, String)] -> IO Bool
+runSuCommand (Just (SuCommand _ cmd ps)) env = boolSystemEnv cmd ps env
+runSuCommand Nothing _ = return False
 
 -- Generates a SuCommand that runs a command as root, fairly portably.
 --
diff --git a/doc/bugs/__47__exe__47__git-annex.mdwn b/doc/bugs/__47__exe__47__git-annex.mdwn
index e1845a75c8..8d7e157f3d 100644
--- a/doc/bugs/__47__exe__47__git-annex.mdwn
+++ b/doc/bugs/__47__exe__47__git-annex.mdwn
@@ -48,3 +48,4 @@ local repository version: 8
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/__47__exe__47__git-annex/comment_2_381ac4e8b27343cdc470584c05edec76._comment b/doc/bugs/__47__exe__47__git-annex/comment_2_381ac4e8b27343cdc470584c05edec76._comment
new file mode 100644
index 0000000000..6db2b4d8bf
--- /dev/null
+++ b/doc/bugs/__47__exe__47__git-annex/comment_2_381ac4e8b27343cdc470584c05edec76._comment
@@ -0,0 +1,7 @@

(Diff truncated)

diff --git a/doc/bugs/__47__exe__47__git-annex/comment_1_8a9e9d83a3dcf07ed76a22f03636f6d1._comment b/doc/bugs/__47__exe__47__git-annex/comment_1_8a9e9d83a3dcf07ed76a22f03636f6d1._comment
new file mode 100644
index 0000000000..45c0e01b52
--- /dev/null
+++ b/doc/bugs/__47__exe__47__git-annex/comment_1_8a9e9d83a3dcf07ed76a22f03636f6d1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-10-26T18:53:40Z"
+ content="""
+Using this command instead should work:
+
+	sudo /home/gyurmo/.local/git-annex.linux/git-annex enable-tor 1000
+"""]]

diff --git a/Annex/WorkTree.hs b/Annex/WorkTree.hs
index e065a2185c..41abc2471e 100644
--- a/Annex/WorkTree.hs
+++ b/Annex/WorkTree.hs
@@ -14,7 +14,7 @@ import Annex.CurrentBranch
 import qualified Database.Keys
 
 {- Looks up the key corresponding to an annexed file in the work tree,
- - by examining what the file links to.
+ - by examining what the symlink points to.
  -
  - An unlocked file will not have a link on disk, so fall back to
  - looking for a pointer to a key in git.
@@ -31,6 +31,16 @@ lookupKey = lookupKey' catkeyfile
 			, catKeyFileHidden file =<< getCurrentBranch
 			)
 
+{- Like lookupKey, but only looks at files staged in git, not at unstaged
+ - changes in the work tree. This means it's slower, but it also has
+ - consistently the same behavior for locked files as for unlocked files.
+ -}
+lookupKeyStaged :: RawFilePath -> Annex (Maybe Key)
+lookupKeyStaged file = catKeyFile file >>= \case
+	Just k -> return (Just k)
+	Nothing -> catKeyFileHidden file =<< getCurrentBranch
+
+{- Like lookupKey, but does not find keys for hidden files. -}
 lookupKeyNotHidden :: RawFilePath -> Annex (Maybe Key)
 lookupKeyNotHidden = lookupKey' catkeyfile
   where
diff --git a/CHANGELOG b/CHANGELOG
index 58972284e3..9926f1ff84 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -15,6 +15,8 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
   * More robust handling of ErrorBusy when writing to sqlite databases.
   * Avoid hanging when a suspended git-annex process is keeping a sqlite
     database locked.
+  * Make --batch mode handle unstaged annexed files consistently
+    whether the file is unlocked or not.
 
  -- Joey Hess <id@joeyh.name>  Mon, 03 Oct 2022 13:36:42 -0400
 
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs
index 0c2617230f..3439b3d580 100644
--- a/CmdLine/Batch.hs
+++ b/CmdLine/Batch.hs
@@ -186,7 +186,7 @@ batchAnnexed fmt seeker keyaction = do
 	matcher <- getMatcher
 	batchFilesKeys fmt $ \(si, v) ->
 		case v of
-			Right f -> lookupKey f >>= \case
+			Right f -> lookupKeyStaged f >>= \case
 				Nothing -> return Nothing
 				Just k -> checkpresent k $
 					startAction seeker si f k
diff --git a/Command/MetaData.hs b/Command/MetaData.hs
index b33e632c73..4568b1f8df 100644
--- a/Command/MetaData.hs
+++ b/Command/MetaData.hs
@@ -155,7 +155,7 @@ parseJSONInput i = case eitherDecode (BU.fromString i) of
 startBatch :: (SeekInput, (Either RawFilePath Key, MetaData)) -> CommandStart
 startBatch (si, (i, (MetaData m))) = case i of
 	Left f -> do
-		mk <- lookupKey f
+		mk <- lookupKeyStaged f
 		case mk of
 			Just k -> go k (mkActionItem (k, AssociatedFile (Just f)))
 			Nothing -> return Nothing
diff --git a/Command/RmUrl.hs b/Command/RmUrl.hs
index c5107bd0eb..efd6e4059d 100644
--- a/Command/RmUrl.hs
+++ b/Command/RmUrl.hs
@@ -47,7 +47,7 @@ batchParser s = case separate (== ' ') (reverse s) of
 			return $ Right (f', reverse ru)
 
 start :: (SeekInput, (FilePath, URLString)) -> CommandStart
-start (si, (file, url)) = lookupKey file' >>= \case
+start (si, (file, url)) = lookupKeyStaged file' >>= \case
 	Nothing -> stop
 	Just key -> do
 		let ai = mkActionItem (key, AssociatedFile (Just file'))
diff --git a/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn
index df111bb55e..944d986dbf 100644
--- a/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn
+++ b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn
@@ -35,3 +35,5 @@ git-annex 10.20221003, provided by datalad/git-annex, on Microsoft Windows Serve
 This affects a hobby project of mine – "gamdam", implemented in [Python](https://github.com/jwodder/gamdam) and [Rust](https://github.com/jwodder/gamdam-rust) — that interacts with git-annex.
 
 [[!meta author=jwodder]]
+
+> [[fixed|done]], see my comments --[[Joey]]
diff --git a/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_2_88b7db5434a56c25c75772caa37bc14a._comment b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_2_88b7db5434a56c25c75772caa37bc14a._comment
new file mode 100644
index 0000000000..80ac4fc644
--- /dev/null
+++ b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_2_88b7db5434a56c25c75772caa37bc14a._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2022-10-26T18:13:42Z"
+ content="""
+I've made --batch handling of unstaged locked files consistent with the
+handling of unstaged unlocked files.
+"""]]

diff --git a/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_1_831dc2185919865d418b29cd06ef42be._comment b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_1_831dc2185919865d418b29cd06ef42be._comment
new file mode 100644
index 0000000000..c1e1d254cb
--- /dev/null
+++ b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work/comment_1_831dc2185919865d418b29cd06ef42be._comment
@@ -0,0 +1,42 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-10-26T16:44:21Z"
+ content="""
+Windows is not needed, this will happen in a 
+repository where `git annex adjust --unlock` has been run.
+
+A simpler example:
+
+	joey@darkstar:~/tmp/t2#master(unlocked)>git-annex addurl --batch
+	http://google.com/
+	addurl http://google.com/
+	(to google.com_) ok
+	^Z
+	joey@darkstar:~/tmp/t2#master(unlocked)>git-annex metadata --batch --json
+	{"file":"google.com_","fields":{"author":["bar"]}}
+
+I'm not sure if this is a bug, because it's documented to output a blank
+line when batch mode is provided a file that is not an annexed file, and
+the file is not an annexed file yet due to the pointer not yet having been
+staged in git. Which is needed, when in an adjusted unlocked branch, for
+git-annex to know that this is an annexed file.
+
+When the file is locked, it just stats the symlink, so the fact that the
+symlink is not yet staged in git doesn't matter.
+
+It does not seem to make sense to have addurl update the index
+after each file it adds, because that would make addurl of a lot
+of files unncessarily slow.
+
+So, I think if anything is changed, it would need to be a change to make
+the behavior with unlocked files consistent with the behavior with locked
+files. Eg, when the symlink is not yet staged in git, treat it as a
+non-annexed file. Which is also consistent with other handling of such
+files by git-annex when not in batch mode.
+
+The solution for your program, though, seems like it will be to end the
+git-annex addurl process before trying to set metadata on just-added files.
+Or, alternatively, to use addurl with --json, extract the key, and set the
+metadata of the key.
+"""]]

diff --git a/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn
new file mode 100644
index 0000000000..df111bb55e
--- /dev/null
+++ b/doc/bugs/addurl_+_metadata_on_Windows_doesn__39__t_work.mdwn
@@ -0,0 +1,37 @@
+(Sorry for the uninformative title, but I had to work within the character limit.)
+
+### Please describe the problem.
+
+`git-annex metadata` does nothing on Windows if invoked while `git-annex addurl` is in progress on other files.
+
+### What steps will reproduce the problem?
+
+On Windows (but not on Linux or macOS, where everything works fine):
+
+- Start `git-annex addurl` in batch mode
+- Feed it two or more URLs
+- After reading the completion message for a URL from addurl's stdout, but before reading the remaining output, run `git-annex metadata` in batch mode and try to set the metadata for the file that was just downloaded.
+- `git-annex metadata` will output an empty line (i.e., just CR LF), and if nothing further is fed to it, it will exit successfully without printing anything else on stdout or stderr.
+- Querying the file's metadata normally after `git-annex addurl` exits will show that no metadata was set for the file.
+
+The Python script at <https://github.com/jwodder/git-annex-bug-20221024/blob/master/mvce.py> (Python 3.8+ required) will run the above steps and show the output from `git-annex metadata`.  A sample run under GitHub Actions can be seen at <https://github.com/jwodder/git-annex-bug-20221024/actions/runs/3322463020/jobs/5491516209>; note the following section of the output under "Run script":
+
+```
+16:04:04 [DEBUG   ] __main__: Opening pipe to: git-annex metadata --batch --json --json-error-messages
+16:04:04 [DEBUG   ] __main__: Input to metadata: b'{"file": "programming/gameboy.pdf", "fields": {"title": ["GameBoy Programming Manual"]}}\n'
+16:04:04 [DEBUG   ] __main__: r.returncode=0
+16:04:04 [DEBUG   ] __main__: r.stdout=b'\r\n'
+16:04:04 [DEBUG   ] __main__: r.stderr=b''
+```
+
+This problem does not always occur, but it seems to occur most of the time.  Using `git-annex registerurl` in place of `git-annex metadata` works fine.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex 10.20221003, provided by datalad/git-annex, on Microsoft Windows Server 2022 
+
+### Please provide any additional information below.
+
+This affects a hobby project of mine – "gamdam", implemented in [Python](https://github.com/jwodder/gamdam) and [Rust](https://github.com/jwodder/gamdam-rust) — that interacts with git-annex.
+
+[[!meta author=jwodder]]

diff --git a/doc/bugs/__47__exe__47__git-annex.mdwn b/doc/bugs/__47__exe__47__git-annex.mdwn
new file mode 100644
index 0000000000..e1845a75c8
--- /dev/null
+++ b/doc/bugs/__47__exe__47__git-annex.mdwn
@@ -0,0 +1,50 @@
+### Please describe the problem.
+
+
+### What steps will reproduce the problem?
+I have system:
+Linux RPI-4B 5.15.74-2-MANJARO-ARM-RPI #1 SMP PREEMPT Thu Oct 20 16:43:17 UTC 2022 aarch64 GNU/Linux
+
+Doesnt't have git-annex in aur and pacman.
+So download git-annex-standalone-arm64.tar.gz from web.
+./runshell
+
+```git-annex enable-tor                                                     (adjusted/master(unlocked)+2) 10:25:22 
+enable-tor 
+  You may be prompted for a password
+
+git-annex: Failed to run as root: /home/gyurmo/.local/git-annex.linux/bin/git-annex enable-tor 1000
+failed
+enable-tor: 1 failed```
+
+
+sudo /home/gyurmo/.local/git-annex.linux/bin/git-annex enable-tor 1000
+[sudo] gyurmo jelszava: 
+/home/gyurmo/.local/git-annex.linux/bin/git-annex: sor: 4: /exe/git-annex: No such file or directory
+
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 10.20220121-g0bcb94487
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux aarch64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 8
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+

diff --git a/doc/forum/Failed_to_push_on_git-lfs/comment_2_52051b251bea7a5ed452769d0320602a._comment b/doc/forum/Failed_to_push_on_git-lfs/comment_2_52051b251bea7a5ed452769d0320602a._comment
new file mode 100644
index 0000000000..61c503889a
--- /dev/null
+++ b/doc/forum/Failed_to_push_on_git-lfs/comment_2_52051b251bea7a5ed452769d0320602a._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="AlexPraga"
+ avatar="http://cdn.libravatar.org/avatar/7c4e10fd352b81279b405f9f5337cdb7"
+ subject="comment 2"
+ date="2022-10-22T15:12:26Z"
+ content="""
+Thanks for answering. I managed to correct it with force-pushing it : `git push lfs main -f`.
+Not sure why it did not work before but it seems to be working now.
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index 3f118af109..58972284e3 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -13,6 +13,8 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
   * When importing from versioned remotes, fix tracking of the content
     of deleted files.
   * More robust handling of ErrorBusy when writing to sqlite databases.
+  * Avoid hanging when a suspended git-annex process is keeping a sqlite
+    database locked.
 
  -- Joey Hess <id@joeyh.name>  Mon, 03 Oct 2022 13:36:42 -0400
 
diff --git a/Database/Handle.hs b/Database/Handle.hs
index 283eef4cf2..84c7623bbf 100644
--- a/Database/Handle.hs
+++ b/Database/Handle.hs
@@ -1,6 +1,6 @@
 {- Persistent sqlite database handles.
  -
- - Copyright 2015-2019 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2022 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -21,6 +21,7 @@ import Utility.Exception
 import Utility.FileSystemEncoding
 import Utility.Debug
 import Utility.DebugLocks
+import Utility.InodeCache
 
 import Database.Persist.Sqlite
 import qualified Database.Sqlite as Sqlite
@@ -38,7 +39,7 @@ import System.IO
 
 {- A DbHandle is a reference to a worker thread that communicates with
  - the database. It has a MVar which Jobs are submitted to. -}
-data DbHandle = DbHandle (Async ()) (MVar Job)
+data DbHandle = DbHandle RawFilePath (Async ()) (MVar Job)
 
 {- Name of a table that should exist once the database is initialized. -}
 type TableName = String
@@ -48,17 +49,17 @@ type TableName = String
 openDb :: RawFilePath -> TableName -> IO DbHandle
 openDb db tablename = do
 	jobs <- newEmptyMVar
-	worker <- async (workerThread (T.pack (fromRawFilePath db)) tablename jobs)
+	worker <- async (workerThread db tablename jobs)
 	
 	-- work around https://github.com/yesodweb/persistent/issues/474
 	liftIO $ fileEncoding stderr
 
-	return $ DbHandle worker jobs
+	return $ DbHandle db worker jobs
 
 {- This is optional; when the DbHandle gets garbage collected it will
  - auto-close. -}
 closeDb :: DbHandle -> IO ()
-closeDb (DbHandle worker jobs) = do
+closeDb (DbHandle _db worker jobs) = do
 	debugLocks $ putMVar jobs CloseJob
 	wait worker
 
@@ -73,7 +74,7 @@ closeDb (DbHandle worker jobs) = do
  - it is able to run.
  -}
 queryDb :: DbHandle -> SqlPersistM a -> IO a
-queryDb (DbHandle _ jobs) a = do
+queryDb (DbHandle _db _ jobs) a = do
 	res <- newEmptyMVar
 	putMVar jobs $ QueryJob $
 		debugLocks $ liftIO . putMVar res =<< tryNonAsync a
@@ -83,22 +84,31 @@ queryDb (DbHandle _ jobs) a = do
 {- Writes a change to the database.
  -
  - Writes can fail when another write is happening concurrently.
- - So write failures are caught and retried repeatedly.
+ - So write failures are caught and retried.
+ -
+ - Retries repeatedly for up to 60 seconds. Part that point, it continues
+ - retrying only if the database shows signs of being modified by another
+ - process at least once each 30 seconds.
  -}
 commitDb :: DbHandle -> SqlPersistM () -> IO ()
-commitDb h wa = robustly (commitDb' h wa)
+commitDb h@(DbHandle db _ _) wa = 
+	robustly (commitDb' h wa) maxretries emptyDatabaseInodeCache
   where
-	robustly :: IO (Either SomeException ()) -> IO ()
-	robustly a = do
+	robustly a retries ic = do
 		r <- a
 		case r of
 			Right _ -> return ()
-			Left _ -> do
-				threadDelay 100000 -- 1/10th second
-				robustly a
+			Left err -> do
+				threadDelay briefdelay
+				retryHelper "write to" err maxretries db retries ic $ 
+					robustly a
+	
+	briefdelay = 100000 -- 1/10th second
+
+	maxretries = 300 :: Int -- 30 seconds of briefdelay
 
 commitDb' :: DbHandle -> SqlPersistM () -> IO (Either SomeException ())
-commitDb' (DbHandle _ jobs) a = do
+commitDb' (DbHandle _ _ jobs) a = do
 	debug "Database.Handle" "commitDb start"
 	res <- newEmptyMVar
 	putMVar jobs $ ChangeJob $
@@ -115,7 +125,7 @@ data Job
 	| ChangeJob (SqlPersistM ())
 	| CloseJob
 
-workerThread :: T.Text -> TableName -> MVar Job -> IO ()
+workerThread :: RawFilePath -> TableName -> MVar Job -> IO ()
 workerThread db tablename jobs = newconn
   where
 	newconn = do
@@ -142,45 +152,47 @@ workerThread db tablename jobs = newconn
 	
 	getjob :: IO (Either BlockedIndefinitelyOnMVar Job)
 	getjob = try $ takeMVar jobs
-	
--- Like runSqlite, but more robust.
---
--- New database connections can sometimes take a while to become usable.
--- This may be due to WAL mode recovering after a crash, or perhaps a
--- situation like described in blob 500f777a6ab6c45ca5f9790e0a63575f8e3cb88f.
--- So, loop until a select succeeds; once one succeeds the connection will
--- stay usable.
---
--- And sqlite sometimes throws ErrorIO when there's not really an IO problem,
--- but perhaps just a short read(). That's caught and retried several times.
-runSqliteRobustly :: TableName -> T.Text -> (SqlPersistM a) -> IO a
+
+{- Like runSqlite, but more robust.
+ -
+ - New database connections can sometimes take a while to become usable,
+ - and selects will fail with ErrorBusy in the meantime. This may be due to
+ - WAL mode recovering after a crash, or a concurrent writer.
+ - So, wait until a select succeeds; once one succeeds the connection will
+ - stay usable.
+ -
+ - Also sqlite sometimes throws ErrorIO when there's not really an IO
+ - problem, but perhaps just a short read(). So also retry on ErrorIO.
+ -
+ - Retries repeatedly for up to 60 seconds. Part that point, it continues
+ - retrying only if the database shows signs of being modified by another
+ - process at least once each 30 seconds.
+ -}
+runSqliteRobustly :: TableName -> RawFilePath -> (SqlPersistM a) -> IO a
 runSqliteRobustly tablename db a = do
-	conn <- opensettle maxretries
-	go conn maxretries
+	conn <- opensettle maxretries emptyDatabaseInodeCache
+	go conn maxretries emptyDatabaseInodeCache
   where
-	maxretries = 100 :: Int
-	
-	rethrow msg e = throwIO $ userError $ show e ++ "(" ++ msg ++ ")"
-	
-	go conn retries = do
+	go conn retries ic = do
 		r <- try $ runResourceT $ runNoLoggingT $
-			withSqlConnRobustly (wrapConnection conn) $
+			withSqlConnRobustly db (wrapConnection conn) $
 				runSqlConn a
 		case r of
 			Right v -> return v
 			Left ex@(Sqlite.SqliteException { Sqlite.seError = e })
-				| e == Sqlite.ErrorIO ->
-					let retries' = retries - 1
-					in if retries' < 1
-						then rethrow "after successful open" ex
-						else go conn retries'
-				| otherwise -> rethrow "after successful open" ex
+				| e == Sqlite.ErrorIO -> do
+					briefdelay
+					retryHelper "access" ex maxretries db retries ic $
+						go conn
+				| otherwise -> rethrow $ errmsg "after successful open" ex
 	
-	opensettle retries = do
-		conn <- Sqlite.open db
-		settle conn retries
+	opensettle retries ic = do
+		conn <- Sqlite.open tdb
+		settle conn retries ic
 
-	settle conn retries = do
+	tdb = T.pack (fromRawFilePath db)
+
+	settle conn retries ic = do
 		r <- try $ do
 			stmt <- Sqlite.prepare conn nullselect
 			void $ Sqlite.step stmt
@@ -188,26 +200,26 @@ runSqliteRobustly tablename db a = do

(Diff truncated)

diff --git a/CHANGELOG b/CHANGELOG
index 6c045d7609..3f118af109 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -12,6 +12,7 @@ git-annex (10.20221004) UNRELEASED; urgency=medium
     by only asking for files under the prefix.
   * When importing from versioned remotes, fix tracking of the content
     of deleted files.
+  * More robust handling of ErrorBusy when writing to sqlite databases.
 
  -- Joey Hess <id@joeyh.name>  Mon, 03 Oct 2022 13:36:42 -0400
 
diff --git a/Database/Handle.hs b/Database/Handle.hs
index ce22d0a661..283eef4cf2 100644
--- a/Database/Handle.hs
+++ b/Database/Handle.hs
@@ -82,22 +82,20 @@ queryDb (DbHandle _ jobs) a = do
 
 {- Writes a change to the database.
  -
- - Writes can fail if another write is happening concurrently.
- - So write failures are caught and retried repeatedly for up to 10
- - seconds, which should avoid all but the most exceptional problems.
+ - Writes can fail when another write is happening concurrently.
+ - So write failures are caught and retried repeatedly.
  -}
 commitDb :: DbHandle -> SqlPersistM () -> IO ()
-commitDb h wa = robustly Nothing 100 (commitDb' h wa)
+commitDb h wa = robustly (commitDb' h wa)
   where
-	robustly :: Maybe SomeException -> Int -> IO (Either SomeException ()) -> IO ()
-	robustly e 0 _ = error $ "failed to commit changes to sqlite database: " ++ show e
-	robustly _ n a = do
+	robustly :: IO (Either SomeException ()) -> IO ()
+	robustly a = do
 		r <- a
 		case r of
 			Right _ -> return ()
-			Left e -> do
+			Left _ -> do
 				threadDelay 100000 -- 1/10th second
-				robustly (Just e) (n-1) a
+				robustly a
 
 commitDb' :: DbHandle -> SqlPersistM () -> IO (Either SomeException ())
 commitDb' (DbHandle _ jobs) a = do
diff --git a/doc/bugs/get_is_busy_doing_nothing/comment_27_45168f110bded2f8c8f9777e1edda945._comment b/doc/bugs/get_is_busy_doing_nothing/comment_27_45168f110bded2f8c8f9777e1edda945._comment
new file mode 100644
index 0000000000..55307eb53d
--- /dev/null
+++ b/doc/bugs/get_is_busy_doing_nothing/comment_27_45168f110bded2f8c8f9777e1edda945._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 27"""
+ date="2022-10-17T18:49:47Z"
+ content="""
+[[todo/withExclusiveLock_blocking_issue]] does not have to be solved for
+every other lock in git-annex first. Since the sqlite database lock would
+be a new lock file, it could use the mtime update method described in there
+without backwards compatibility issues.
+
+ErrorBusy can also occur when opening a new database connection for read,
+but it retries that as often as necessary. Which does mean that suspending
+git-annex at just the wrong time can already cause other git-annex
+processes to stall forever waiting to read from the database. 
+
+So, in a way, it would be ok for write to also retry each time it gets
+ErrorBusy, rather than the current limited number of retries. If that does
+cause git-annex to block when another git-annex process is suspended, it
+would not be a new behavior.
+
+Also, the mtime file method described in
+[[todo/withExclusiveLock_blocking_issue]] could be used without a lock file
+in order to detect when a suspended process is causing ErrorBusy. And can
+avoid that situation for both writes and reads.
+
+So, plan: 
+
+1. Retry forever on ErrorBusy when writing to sqlite database.
+   (I've made this change now... So I think probably this bug can't
+   occur any longer.)
+2. While running opensettle and ChangeJob, have a background thread that 
+   periodically updates a mtime file.
+3. If ErrorBusy is received repeatedly for some amount of time,
+   check the mtime file. If it's not being updated, give up, since
+   a suspended git-annex process apparently has the sqlite database locked.
+"""]]
diff --git a/doc/todo/withExclusiveLock_blocking_issue.mdwn b/doc/todo/withExclusiveLock_blocking_issue.mdwn
index 6915015da3..32c750ed7b 100644
--- a/doc/todo/withExclusiveLock_blocking_issue.mdwn
+++ b/doc/todo/withExclusiveLock_blocking_issue.mdwn
@@ -1,20 +1,83 @@
-Some parts of git-annex use withExclusiveLock or otherwise wait for an
-exclusive lock and hold it while performing an operation. Now consider what
-happens if the git-annex process is suspended. Another git-annex process
-that is running and that blocks on the same lock will stall forever, until
-the git-annex process is resumed.
+Some parts of git-annex wait for an exclusive lock, and once they take it,
+hold it while performing an operation. Now consider what happens if the
+git-annex process is suspended. Another git-annex process that is running
+and that waits to take the same exclusive lock (or a shared lock of the
+same file) will stall forever, until the git-annex process is resumed.
 
 These time windows tend to be small, but may not always be.
 
-Would it be better for the second git-annex process, rather than hanging
-indefinitely, to try to take the lock a few times over a few seconds, and
-then error out? The risk with doing that is, when 2 concurrent git-annex
-processes are running and taking the locks repeatedly, one might get
-unlucky, fail to take the lock, and error out, when waiting a little longer
-would have succeeded, because the other process is not holding the lock all
-the time.
-
 Is there any better way git-annex could handle this? Is it a significant
 problem at all? I don't think I've ever seen it happen, but I rarely ^Z
 git-annex either. How do other programs handle this, if at all?
 --[[Joey]]
+
+----
+
+Would it be better for the second git-annex process, rather than hanging
+indefinitely, to timeout after a few seconds?
+
+But how many seconds? What if the system is under heavy load?
+
+> What could be done is, update the lock's file's mtime after successfully
+> taking the lock. Then, as long as the mtime is advancing, some other
+> process is actively using it, and it's ok for our process to wait
+> longer.
+> 
+> (Updating the mtime would be a problem when locking annex object files
+> in v9 and earlier. Luckily, that locking is not done with a blocking
+> lock anyway.)
+
+> If the lock file's mtime is being checked, the process that is
+> blocking with the lock held could periodically update the mtime.
+> A background thread could manage that. If that's done every ten seconds,
+> then an mtime more than 20 seconds old indicates that the lock is
+> held by a suspended process. So git-annex would stall for up to 20-30
+> seconds before erroring out when a lock is held by a suspended process.
+> That seems acceptible, it doesn't need to deal with this situation
+> instantly, it just needs to not block indefinitely. And updating the
+> mtime every 10 seconds should not be too much IO.
+> 
+> When an old version of git-annex has the lock held, it won't be updating
+> the mtime. So if it takes longer than 10 seconds to do the operation with
+> the lock held, a new version may complain that it's suspended when it's
+> really not. This could be avoided by checking what process holds the
+> lock, and whether it's suspended. But probably 10 seconds is enough
+> time for all the operations git-annex takes a blocking lock for
+> currently to finish, and if so we don't need to worry about this situation?
+> 
+> >  Unfortunately not: importKeys takes an exclusive lock and holds it while
+> > downloading all the content! This seems like a bug though, because it can
+> > cause other git-annex processes that are eg storing content in a remote
+> > to block for a long time.
+> > 
+> > Another one is Database.Export.writeLockDbWhile, which takes an
+> > exclusive lock while running eg, Command.Export.changeExport,
+> > which may sometimes need to do a lot of work.
+> >
+> > Another one is Annex.Queue.flush, which probably mostly runs in under
+> > 10 seconds, but maybe not always, and when annex.queuesize is changed,
+> > could surely take longer.
+> 
+> To avoid problems when old git-annex's are also being used, it could
+> update and check the mtime of a different file than the lock file.
+> 
+> Start by trying to take the lock for up to 10 seconds. If it takes the
+> lock, create the mtime file and start a thread that updates the mtime 
+> every 10 seconds until the lock is closed, and delete the mtime file
+> before closing the lock handle. 
+> 
+> When it times out taking the lock, if the mtime file does not exist, an
+> old git-annex has the lock; if the mtime file does exist, then check
+> if its timestamp has advanced; if not then a new git-annex has the lock
+> and is suspended and it can error out.
+> 
+> Oops: There's a race in the method above; a timeout may occur
+> right when the other process has taken the lock, but has not updated
+> the mtime file yet. Then that process would incorrectly be treated
+> as an old git-annex process.
+> 
+> So: To support old git-annex, it seems it will need to check, when the
+> lock is held, what process has the lock. And then check if that process
+> is suspended or not. Which means looking in /proc. Ugh.
+> 
+> Or: Change to checking lock mtimes only in git-annex v11..

diff --git a/doc/forum/Failed_to_push_on_git-lfs/comment_1_f83dbcfdd09de3ce908c0d4a9daef458._comment b/doc/forum/Failed_to_push_on_git-lfs/comment_1_f83dbcfdd09de3ce908c0d4a9daef458._comment
new file mode 100644
index 0000000000..f3c9d06728
--- /dev/null
+++ b/doc/forum/Failed_to_push_on_git-lfs/comment_1_f83dbcfdd09de3ce908c0d4a9daef458._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2022-10-17T16:28:48Z"
+ content="""
+This doesn't involve LFS at all, it's a regular git branch being pushed in
+the regular way. So you can certianly solve the problem with some
+combination of `git pull`, `git merge`, and `git push`.
+
+That said, I don't know why `git-annex sync` didn't work in your situation.
+I created some test git-lfs repos on github and never saw any difficulty
+syncing with them.
+"""]]