diff --git a/doc/bugs/Build_failing_on_Windows_since_ef3ab0769.mdwn b/doc/bugs/Build_failing_on_Windows_since_ef3ab0769.mdwn
new file mode 100644
index 0000000000..842869ccec
--- /dev/null
+++ b/doc/bugs/Build_failing_on_Windows_since_ef3ab0769.mdwn
@@ -0,0 +1,74 @@
+Due to the type change of `tryMakeLockHandle` in commit ef3ab0769, compilation of git-annex is currently failing on Windows with the below error.  I am unsure of the best way to fix it.
+
+[[!format txt """
+Utility\LockPool\Windows.hs:25:19: error:
+    * Couldn't match type `(LockHandle, t1)' with `LockHandle'
+      Expected type: IO (Maybe LockHandle)
+        Actual type: IO (Maybe (LockHandle, t1))
+    * In the expression:
+        tryMakeLockHandle
+          P.lockPool file (\ p f -> P.tryTakeLock p f LockShared)
+          (\ f _ -> fmap mk <$> F.lockShared f)
+      In an equation for `lockShared':
+          lockShared file
+            = tryMakeLockHandle
+                P.lockPool file (\ p f -> P.tryTakeLock p f LockShared)
+                (\ f _ -> fmap mk <$> F.lockShared f)
+   |
+25 | lockShared file = tryMakeLockHandle P.lockPool file
+
+   |                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^...
+
+Utility\LockPool\Windows.hs:27:18: error:
+    * Couldn't match type `FileLockOps' with `(FileLockOps, t1)'
+      Expected type: IO (Maybe (FileLockOps, t1))
+        Actual type: IO (Maybe FileLockOps)
+    * In the expression: fmap mk <$> F.lockShared f
+      In the fourth argument of `tryMakeLockHandle', namely
+        `(\ f _ -> fmap mk <$> F.lockShared f)'
+      In the expression:
+        tryMakeLockHandle
+          P.lockPool file (\ p f -> P.tryTakeLock p f LockShared)
+          (\ f _ -> fmap mk <$> F.lockShared f)
+   |
+27 |         (\f _ -> fmap mk <$> F.lockShared f)
+
+   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Utility\LockPool\Windows.hs:36:22: error:
+    * Couldn't match type `(LockHandle, t0)' with `LockHandle'
+      Expected type: IO (Maybe LockHandle)
+        Actual type: IO (Maybe (LockHandle, t0))
+    * In the expression:
+        tryMakeLockHandle
+          P.lockPool file (\ p f -> P.tryTakeLock p f LockExclusive)
+          (\ f _ -> fmap mk <$> F.lockExclusive f)
+      In an equation for `lockExclusive':
+          lockExclusive file
+            = tryMakeLockHandle
+                P.lockPool file (\ p f -> P.tryTakeLock p f LockExclusive)
+                (\ f _ -> fmap mk <$> F.lockExclusive f)
+   |
+36 | lockExclusive file = tryMakeLockHandle P.lockPool file
+
+   |                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^...
+
+Utility\LockPool\Windows.hs:38:18: error:
+    * Couldn't match type `FileLockOps' with `(FileLockOps, t0)'
+      Expected type: IO (Maybe (FileLockOps, t0))
+        Actual type: IO (Maybe FileLockOps)
+    * In the expression: fmap mk <$> F.lockExclusive f
+      In the fourth argument of `tryMakeLockHandle', namely
+        `(\ f _ -> fmap mk <$> F.lockExclusive f)'
+      In the expression:
+        tryMakeLockHandle
+          P.lockPool file (\ p f -> P.tryTakeLock p f LockExclusive)
+          (\ f _ -> fmap mk <$> F.lockExclusive f)
+   |
+38 |         (\f _ -> fmap mk <$> F.lockExclusive f)
+
+   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+"""]]
+
+[[!meta author=jwodder]]
+[[!tag projects/datalad]]

diff --git a/doc/forum/conflict_with_git-crypt/comment_2_cda648fcc5f213af06fe98428910f561._comment b/doc/forum/conflict_with_git-crypt/comment_2_cda648fcc5f213af06fe98428910f561._comment
new file mode 100644
index 0000000000..cf4a401e89
--- /dev/null
+++ b/doc/forum/conflict_with_git-crypt/comment_2_cda648fcc5f213af06fe98428910f561._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="alt"
+ subject="comment 2"
+ date="2021-12-08T20:01:33Z"
+ content="""
+Thank you for your explanation! 
+
+Removing `* filter=annex` altogether causes problems when the repo has unlocked files, but removing it from `.git/info/attributes` and adding a `.gitattributes` file with the following contents at the repository root seems to allow both git-crypt and git-annex to work without issue:
+
+```
+* filter=annex
+my_secret filter=git-crypt diff=git-crypt
+```
+
+"""]]

diff --git a/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_2_159fcb2296537d29a291e32bac307ef1._comment b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_2_159fcb2296537d29a291e32bac307ef1._comment
new file mode 100644
index 0000000000..9adc073eee
--- /dev/null
+++ b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_2_159fcb2296537d29a291e32bac307ef1._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-12-08T19:19:38Z"
+ content="""
+Seems this breakage is due to https://github.com/ghc-proposals/ghc-proposals/blob/master/proposals/0287-simplify-subsumption.rst
+
+Fixed it.
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index 69d2da9f24..15b6c1647b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -8,6 +8,7 @@ git-annex (8.20211124) UNRELEASED; urgency=medium
     already exported.
   * Fix locking problems when annex.pidlock is set and concurrency is
     enabled eg with -J.
+  * Fix build with ghc 9.0.1
 
  -- Joey Hess <id@joeyh.name>  Tue, 23 Nov 2021 15:58:27 -0400
 
diff --git a/Remote/Glacier.hs b/Remote/Glacier.hs
index eaf0a5cc82..37d4f1e44f 100644
--- a/Remote/Glacier.hs
+++ b/Remote/Glacier.hs
@@ -28,6 +28,7 @@ import Utility.Metered
 import Annex.UUID
 import Utility.Env
 import Types.ProposedAccepted
+import Utility.Hash (IncrementalVerifier)
 
 type Vault = String
 type Archive = FilePath
@@ -175,7 +176,7 @@ store' r k b p = go =<< glacierEnv c gc u
 		forceSuccessProcess cmd pid
 	go' _ _ _ _ _ = error "internal"
 
-retrieve :: Remote -> Retriever
+retrieve :: forall a. Remote -> Key -> MeterUpdate -> Maybe IncrementalVerifier -> (ContentSource -> Annex a) -> Annex a
 retrieve = byteRetriever . retrieve'
 
 retrieve' :: forall a. Remote -> Key -> (L.ByteString -> Annex a) -> Annex a
diff --git a/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn
index 758defb8cf..4044f561c4 100644
--- a/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn
+++ b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn
@@ -50,3 +50,4 @@ make: *** [Makefile:58: git-annex] Error 1
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 Oh absolutely. This version compiles fine under alpine 5.14, which uses GHC 8.8.4. I've been using this software for many years without major issues, and it's a linchpin of my backup infrastructure. Thanks for the all the good work!
 
+> [[fixed|done]] --[[Joey]]

diff --git a/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_3_4c957c33a1c8dc179e0e6cd67a5a29d9._comment b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_3_4c957c33a1c8dc179e0e6cd67a5a29d9._comment
new file mode 100644
index 0000000000..b0837ff5eb
--- /dev/null
+++ b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_3_4c957c33a1c8dc179e0e6cd67a5a29d9._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2021-12-08T18:56:51Z"
+ content="""
++1 on the idea of \"combine them in memory the same as if a merge had been done.\" -- sounds great to me, even though indeed might cost some performance.  I would assume that with `annex.merge-annex-branches=false` such operation would be skipped, and then read-only operations would operate on current, possibly lacking some merges git-annex branch
+
+re \"wanted\": tried on the same repository, but it was updated so its git-annex branch is now up to date with the remote one... For now just managed to unravel a more of a UX issue: [https://git-annex.branchable.com/bugs/wanted_on_read-only_repo_crashes_/?updated](https://git-annex.branchable.com/bugs/wanted_on_read-only_repo_crashes_/?updated)
+"""]]

diff --git a/doc/bugs/wanted_on_read-only_repo_crashes_.mdwn b/doc/bugs/wanted_on_read-only_repo_crashes_.mdwn
new file mode 100644
index 0000000000..244b2a1c89
--- /dev/null
+++ b/doc/bugs/wanted_on_read-only_repo_crashes_.mdwn
@@ -0,0 +1,58 @@
+### Please describe the problem.
+
+follow up on this particular issue from [https://git-annex.branchable.com/bugs/merge-annex-branches__61__false_-_automate_and_extend/#comment-5d414fbc11cff204b9bcc93685e3bbcb](https://git-annex.branchable.com/bugs/merge-annex-branches__61__false_-_automate_and_extend/#comment-5d414fbc11cff204b9bcc93685e3bbcb) . 
+
+### What steps will reproduce the problem?
+
+Couldn't reproduce on original machine/repo since that one was updated and same error does not appear any longer.  
+
+So did from scratch on some arbitrary repo but caught another "problem" (or not -- for you to decide) that `annex` would crash if I did not init it yet:
+
+
+```shell
+lena:/tmp
+$> umask 022
+1 28194.....................................:Wed 08 Dec 2021 01:49:40 PM EST:.
+lena:/tmp
+$> git clone http://datasets.datalad.org/repronim/containers/.git
+Cloning into 'containers'...
+remote: Enumerating objects: 6123, done.
+remote: Counting objects: 100% (6123/6123), done.
+remote: Compressing objects: 100% (3691/3691), done.
+remote: Total 6123 (delta 3353), reused 2747 (delta 1502)
+Receiving objects: 100% (6123/6123), 609.01 KiB | 4.61 MiB/s, done.
+Resolving deltas: 100% (3353/3353), done.
+1 28195.....................................:Wed 08 Dec 2021 01:49:43 PM EST:.
+lena:/tmp
+$> cd containers 
+LICENSE  README.md  artwork/  binds/  images/  licenses/  scripts/  travis/
+1 28196.....................................:Wed 08 Dec 2021 01:49:46 PM EST:.
+(git)lena:/tmp/containers[master]
+$> git update-ref refs/remotes/origin/git-annex refs/remotes/origin/git-annex^
+1 28197.....................................:Wed 08 Dec 2021 01:49:53 PM EST:.
+(git)lena:/tmp/containers[master]
+$> sudo -u nobody git -c annex.merge-annex-branches=false annex wanted here   
+fatal: cannot lock ref 'refs/heads/git-annex': Unable to create '/tmp/containers/.git/refs/heads/git-annex.lock': Permission denied
+error: could not lock config file .git/config: Permission denied
+git-annex: git [Param "config",Param "user.name",Param "nobody"] failed
+CallStack (from HasCallStack):
+  error, called at ./Git/Command.hs:42:17 in main:Git.Command
+1 28198 ->1.....................................:Wed 08 Dec 2021 01:49:58 PM EST:.
+(git)lena:/tmp/containers[master]git
+$> sudo -u nobody git annex wanted here         
+fatal: cannot lock ref 'refs/heads/git-annex': Unable to create '/tmp/containers/.git/refs/heads/git-annex.lock': Permission denied
+error: could not lock config file .git/config: Permission denied
+git-annex: git [Param "config",Param "user.name",Param "nobody"] failed
+CallStack (from HasCallStack):
+  error, called at ./Git/Command.hs:42:17 in main:Git.Command
+1 28199 ->1.....................................:Wed 08 Dec 2021 01:50:07 PM EST:.
+(git)lena:/tmp/containers[master]git
+$> git annex version
+git-annex version: 8.20211123
+...
+```
+
+the issue is different from original but still might be worth addressing one way or another (e.g. by providing a specific informative message that it was not yet init-ed)
+
+[[!meta author=yoh]]
+[[!tag projects/datalad]]

diff --git a/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_1_c3fbb6eb15a510ae9b1af13cd7df028a._comment b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_1_c3fbb6eb15a510ae9b1af13cd7df028a._comment
new file mode 100644
index 0000000000..7cfba7c9f0
--- /dev/null
+++ b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_1_c3fbb6eb15a510ae9b1af13cd7df028a._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-12-08T17:21:42Z"
+ content="""
+Firstly, you're overstating the problem. This does not affect queries in
+all read-only repositories. If the read-only repository does not have any
+remote git-annex branches that have not been merged into the local
+branch, the query will succeed. Only when there is such an unmerged branch
+will there be a permission error when it merges the annex branch.
+
+So, if you eg use `git anenx sync`, you would never end up in a situation
+where annex.merge-annex-branches is needed, because the remote git-annex
+branches are always merged by it.
+
+Setting annex.merge-annex-branches false manually is one thing,
+you know you are changing its behavior. Setting it implicitly due to
+permissions is another thing, what would result in git-annex query commands
+changing their behavior based on permissions. The previously linked thread
+discussed and rejected that idea, and rightly so.
+
+What I would be willing to do would be to have git-annex detect when it
+cannot merge remote annex branches, and skip the merging, and instead when
+querying the value of a file from the git-annex branch, also query the
+values from the unmerged remote branches, and combine them in memory the
+same as if a merge had been done. This would make git-annex query
+operations slower in this unusual situation, but they would always
+have the same result as if it was able to perform the merge.
+"""]]
diff --git a/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_2_c33bf0bdf6bf85e4ee9e1286208c0b73._comment b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_2_c33bf0bdf6bf85e4ee9e1286208c0b73._comment
new file mode 100644
index 0000000000..69046ec7aa
--- /dev/null
+++ b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend/comment_2_c33bf0bdf6bf85e4ee9e1286208c0b73._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-12-08T17:38:54Z"
+ content="""
+I have not been able to reproduce the 
+`git -c annex.merge-annex-branches=false annex wanted here`
+problem. You apparently have annex.pidlock set since
+"locktmp" is used when doing pid locks, but despite setting that and having
+an unmerged remote git-annex branch, this is as close as I can get:
+
+	nobody@darkstar:/tmp/yyy$ git-annex wanted here
+	git-annex: /tmp/yyy/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+	wanted: 1 failed
+	nobody@darkstar:/tmp/yyy$ git -c annex.merge-annex-branches=false annex wanted here
+
+	nobody@darkstar:/tmp/yyy$
+
+Anyway, this is an entirely different problem and should be in its own bug
+report, once you determine how to reproduce it..
+"""]]

diff --git a/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism/comment_1_7078d380be9e78fea9fd5bec4ef9594b._comment b/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism/comment_1_7078d380be9e78fea9fd5bec4ef9594b._comment
new file mode 100644
index 0000000000..1f16892f83
--- /dev/null
+++ b/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism/comment_1_7078d380be9e78fea9fd5bec4ef9594b._comment
@@ -0,0 +1,36 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-12-08T16:48:15Z"
+ content="""
+But git-annex does allow for multiple special remotes of the same type
+that need different creds. At least for S3, when you run `git-annex
+initremote` it copies the env var values into a file, so you only need to
+have the env var set when running initremote, and all subsequent use of
+the remote will use the value it cached (unless you keep the env vars set,
+then it does use the env var).
+
+And there is a UI to prompt the user for the creds:
+
+	joey@darkstar:/tmp/foo>git annex initremote s3 type=S3  encryption=none
+	initremote s3 (checking bucket...)
+	  Set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY to use S3
+	git-annex: No S3 credentials configured
+	failed
+
+Perhaps initremote of S3/glacier/webdav remotes could query git credential when
+the env vars are not set. Those are the only 3 built-in remotes that use the
+env var creds system actually.
+
+Fundamentally, what kind of authentication information a special remote needs
+depends on the service it's talking to, and there is a very large scope
+of possibilities, not only the simple username+password that git credential
+is designed for. Even S3 has a third possible credential that
+is sometimes needed and would not fit into git credential (`AWS_SESSION_TOKEN`).
+Other remotes such a borg don't even use git-annex to handle credentials,
+but run a program that prompts for whatever it needs, or rely on
+other stored data like ssh keys. A remote could just as well need a hardware
+token, or one time password, or biometric id, whatever. So git-annex
+has to leave authentication up to the remotes to prompt for in whatever
+way makes sense for them, aside from the simple username+password case.
+"""]]

diff --git a/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_4_a5822d922a107e2a0e8972b7f55d92cc._comment b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_4_a5822d922a107e2a0e8972b7f55d92cc._comment
new file mode 100644
index 0000000000..8b1b3a1b90
--- /dev/null
+++ b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_4_a5822d922a107e2a0e8972b7f55d92cc._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2021-12-08T16:42:47Z"
+ content="""
+Your use of `git-annex sync --no-content` doesn't make much sense,
+of course that will not cause it to export any content because you told it
+not to. Also there is configuration needed for git-annex sync to export
+a branch to a remote and you have not shown that you performed such
+configuration, and the output of the command seems to indicate you have
+not.
+
+Your `git annex export master --to film-depot` seems to have behaved
+correctly, since the content of some files were not present it was not able
+to export those. Running `git annex get` and then re-running that command
+will make update the export with the missing files. I have tested that.
+"""]]

diff --git a/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_3_b7dcb0a5e69e620278d1aa4d6f5829cd._comment b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_3_b7dcb0a5e69e620278d1aa4d6f5829cd._comment
new file mode 100644
index 0000000000..268ee3fa25
--- /dev/null
+++ b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_3_b7dcb0a5e69e620278d1aa4d6f5829cd._comment
@@ -0,0 +1,79 @@
+[[!comment format=mdwn
+ username="jasonb@ab4484d9961a46440958fa1a528e0fc435599057"
+ nickname="jasonb"
+ avatar="http://cdn.libravatar.org/avatar/c7330f4da122c671b935fc1d58bb02b1"
+ subject="comment 3"
+ date="2021-12-08T16:25:31Z"
+ content="""
+```
+git annex sync --no-content --no-push --no-pull
+commit
+On branch master
+nothing to commit, working tree clean
+ok
+
+git log -n3
+commit 9d18fec41a08dbcd0377a792d987be7c158fedbc (HEAD -> master, film-depot/master, MBP/synced/master, MBP/master, synced/master)
+```
+
+Originally, it seemed to work and exported the files that I expected; What confused me is there were not in the directory on the export target.
+
+```
+git annex export master --to film-depot
+export film-depot .gitattributes ok
+export film-depot .gitignore ok
+export film-depot 00-GIT-ANNEX ok
+export film-depot American Pie 2 (2001) 720p.mp4 ok
+export film-depot Armageddon (1998) 1080p.mp4 (not available) failed
+export film-depot Avengers, Age of Ultron (2015) 1080p.mp4 ok
+export film-depot Beverly Hills Cop 2 (1987) 1080p.mp4 (not available) failed
+export film-depot Beverly Hills Cop III (1994) 1080p.mp4 (not available) failed
+export film-depot Final Fantasy The Spirits Within (2001) 1080p.mkv (not available) failed
+export film-depot Groundhog Day (1993) 1080p.mp4 (not available) failed
+export film-depot In the Heat of the Night (1967) 720p.mkv ok
+export film-depot Indiana Jones and the Raiders of the Lost Ark (1981) 1080p.mp4
+ok
+export film-depot Inglourious Basterds (2009) 1080p.mkv
+ok
+export film-depot Interstellar (2014) 1080p.mp4 (not available) failed
+export film-depot King Arthur (2004).mkv ok
+export film-depot Last of the Mohicans, The (1992).mkv ok
+export film-depot Lethal Weapon 2 (1989) 720p.mkv
+ok
+export film-depot Pirates of the Caribbean Curse of the Black Pearl (2003) 1080p.mp4
+ok
+export film-depot README.md ok
+export film-depot Rock, The (1996).mkv ok
+export film-depot Rogue One A Star Wars Story (2016) 1080p.mkv
+ok
+export film-depot Star Trek II The Wrath of Khan (1982) 1080p.mkv
+ok
+export film-depot Star Trek III The Search for Spock (1984) 1080p.mkv
+ok
+export film-depot Star Trek IV The Voyage Home (1986) 1080p.mkv
+ok
+export film-depot Star Trek The Motion Picture (1979) 1080p.mkv
+ok
+export film-depot Star Trek V The Final Frontier (1989) 1080p.mkv
+ok
+export film-depot Star Trek VI The Undiscovered Country (1991) 1080p.mkv
+ok
+export film-depot Terminator Genisys (2015) 1080p.mp4 (not available) failed
+export film-depot The Matrix (1999) 1080p.mp4
+ok
+export film-depot The Mummy (1999).mkv ok
+export film-depot The Patriot (2000).mkv ok
+export film-depot Underworld (2003) 1080p.mkv
+ok
+(recording state in git...)
+export: 7 failed
+```
+
+So I went and recovered those files from another annex that had them, and re-ran the export. No errors.
+
+But also, it didn't export the files that it couldn't find; It's like after the first failed attempt, that was it. Even with the files present, it did not export them.
+
+So that's what led to the `rm -Rf` and finally the `cp`.
+
+Maybe I can reset it all by reinitializing the export remote somehow?
+"""]]

diff --git a/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_2_57ce71a398b0e2c947164e97ba8d0e2d._comment b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_2_57ce71a398b0e2c947164e97ba8d0e2d._comment
new file mode 100644
index 0000000000..443b20ab4d
--- /dev/null
+++ b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_2_57ce71a398b0e2c947164e97ba8d0e2d._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="jasonb@ab4484d9961a46440958fa1a528e0fc435599057"
+ nickname="jasonb"
+ avatar="http://cdn.libravatar.org/avatar/c7330f4da122c671b935fc1d58bb02b1"
+ subject="comment 2"
+ date="2021-12-08T00:46:39Z"
+ content="""
+Oh, so in my haste to complete the task, I `rm -Rf *` the target directory, and when the files still didn't copy, and just ran `cp`. I don't know if I still have output to that effect, though. But I did have quite a few files with new names, so I was puzzled that those files were not copied because no files with those names or hashes had ever been exported to this annex remote.
+
+I don't know if there is some local database or log of what git annex thinks is on a special remote that I can paste to demonstrate the discrepancy?
+
+Thanks.
+"""]]

diff --git a/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism.mdwn b/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism.mdwn
new file mode 100644
index 0000000000..13c8315ddf
--- /dev/null
+++ b/doc/todo/more_flexible___40____34__external__34____63____41___credentials_mechanism.mdwn
@@ -0,0 +1,13 @@
+ATM git-annex heavily relies on credentials needed by special remote(s) (S3, WEBDAV) to be passed via environment variables. 
+
+- main shortcoming: those environment variables (e.g. AWS_ACCESS_KEY_ID) do not allow for different set of credentials to be provided to be used e.g. by multiple special remotes of the same `type` and would require multiple git-annex invocations to get through all of them
+- user experience:
+  - there is no UI to prompt user to enter needed/desired secrets.  That also relates to software relying on git-annex, such as datalad, as it somehow needs to know/discover that underlying call to `git-annex` apparently needs some credentials to be passed via env vars
+  - users then (typically)  need to hardcode values of those secrets verbatim in their scripts (instead of using some more secure credentials store)
+
+Given those (and possibly other) shortcomings, I thought it would have been great if git-annex supported some "credentials" storage/prompting/querying mechanism which would address main shortcoming and improve user experience. May be git-annex could use [git-credential](https://git-scm.com/docs/git-credential) mechanism. If not, then may be have some protocol created to allow for external git-annex-credential-* helpers?
+
+For reference, here is a PR in DataLad [https://github.com/datalad/datalad/pull/5796](https://github.com/datalad/datalad/pull/5796) to provide `git-credential-datalad` helper so `git` could query our (datalad) credentials store.
+
+[[!meta author=yoh]]
+[[!tag projects/datalad]]

diff --git a/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_1_70a68381641ea3369cd19e2864de753c._comment b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_1_70a68381641ea3369cd19e2864de753c._comment
new file mode 100644
index 0000000000..7fe99b479c
--- /dev/null
+++ b/doc/forum/So_this_is_embarrassing__59___git_export_not_working/comment_1_70a68381641ea3369cd19e2864de753c._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-12-07T17:01:34Z"
+ content="""
+If you have already exported a file to a special remote, doing an export
+again will skip that same file and not display anything[1]. So without you
+showing me anything else, it looks like that is what the situation is, and
+nothing is wrong. If you think something is wrong, you'll need to explain
+in more detail what it is.
+
+----
+
+[1] Except sometimes for file that are checked into git and not git-annex, 
+which explains why only .gitignore and README and such are displated.
+"""]]

diff --git a/doc/forum/conflict_with_git-crypt/comment_1_b81a3cd99ff04e21ef520a497c0bd46a._comment b/doc/forum/conflict_with_git-crypt/comment_1_b81a3cd99ff04e21ef520a497c0bd46a._comment
new file mode 100644
index 0000000000..bdef755ec0
--- /dev/null
+++ b/doc/forum/conflict_with_git-crypt/comment_1_b81a3cd99ff04e21ef520a497c0bd46a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-12-07T16:55:51Z"
+ content="""
+I guess this is about them both setting the filter attribute.
+
+git-annex does not write to .git/info/attributes after
+`git-annex init` so you can move the filter=annex line from there to
+another file, or change the line to only match on some files not `*`,
+or even delete it, if you want to.
+"""]]

diff --git a/doc/forum/So_this_is_embarrassing__59___git_export_not_working.mdwn b/doc/forum/So_this_is_embarrassing__59___git_export_not_working.mdwn
new file mode 100644
index 0000000000..e2cc928c00
--- /dev/null
+++ b/doc/forum/So_this_is_embarrassing__59___git_export_not_working.mdwn
@@ -0,0 +1,243 @@
+Always worked before, doesn't work now. I just get this.
+
+```
+git annex export master --to film-depot --verbose
+export film-depot .gitattributes ok
+export film-depot .gitignore ok
+export film-depot 00-GIT-ANNEX ok
+export film-depot README.md ok
+(recording state in git...)
+```
+
+Previously I had 99 movie files that would export to my external drive. Now no dice.
+
+```
+[2021-12-06 22:22:25.524608] (Utility.Process) process [72333] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","git-annex"]
+[2021-12-06 22:22:25.529718] (Utility.Process) process [72333] done ExitSuccess
+[2021-12-06 22:22:25.530949] (Utility.Process) process [72334] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2021-12-06 22:22:25.536689] (Utility.Process) process [72334] done ExitSuccess
+[2021-12-06 22:22:25.537822] (Utility.Process) process [72335] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..59ceb4035e85ed634375596e4a44c5b3ad300652","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.5423] (Utility.Process) process [72335] done ExitSuccess
+[2021-12-06 22:22:25.5431] (Utility.Process) process [72336] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..8c7c4c0941afc7d9a05294dad06414fd0333eba9","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.549488] (Utility.Process) process [72336] done ExitSuccess
+[2021-12-06 22:22:25.550436] (Utility.Process) process [72337] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..f442bffc235f879ec4fb31dfcbf89b3ac778b520","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.5569] (Utility.Process) process [72337] done ExitSuccess
+[2021-12-06 22:22:25.557738] (Utility.Process) process [72338] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..eb25a989c75c43bc36c99c4ccf28ef17123a1f5e","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.566045] (Utility.Process) process [72338] done ExitSuccess
+[2021-12-06 22:22:25.56697] (Utility.Process) process [72339] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..6eda2fa9081d65baf6758e608fad471fd0e9dc3e","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.576185] (Utility.Process) process [72339] done ExitSuccess
+[2021-12-06 22:22:25.577034] (Utility.Process) process [72340] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..89670c26510137760b55e0cc3b8378968e9f7b01","--pretty=%H","-n1"]
+[2021-12-06 22:22:25.584382] (Utility.Process) process [72340] done ExitSuccess
+[2021-12-06 22:22:25.591735] (Utility.Process) process [72341] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"]
+[2021-12-06 22:22:25.592642] (Utility.Process) process [72342] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2021-12-06 22:22:25.597233] (Utility.Process) process [72343] read: git ["config","--null","--list"]
+[2021-12-06 22:22:25.600309] (Utility.Process) process [72343] done ExitSuccess
+[2021-12-06 22:22:25.601475] (Utility.Process) process [72344] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","rev-parse","--verify","--quiet","master:"]
+[2021-12-06 22:22:25.604843] (Utility.Process) process [72344] done ExitSuccess
+[2021-12-06 22:22:25.607159] (Utility.Process) process [72345] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","mktree","--batch","-z"]
+[2021-12-06 22:22:25.607845] (Utility.Process) process [72346] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-tree","--full-tree","-z","-r","-t","--","bf9aa984fda078c0f7e2ae3f416f7366a1bd6906"]
+[2021-12-06 22:22:25.613684] (Utility.Process) process [72346] done ExitSuccess
+[2021-12-06 22:22:25.613994] (Utility.Process) process [72345] done ExitSuccess
+[2021-12-06 22:22:25.615026] (Utility.Process) process [72347] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/master"]
+[2021-12-06 22:22:25.62044] (Utility.Process) process [72347] done ExitSuccess
+[2021-12-06 22:22:25.625281] (Annex.Branch) read export.log
+[2021-12-06 22:22:25.626277] (Annex.Branch) set export.log
+[2021-12-06 22:22:25.627203] (Utility.Process) process [72348] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2021-12-06 22:22:25.632737] (Utility.Process) process [72348] done ExitSuccess
+[2021-12-06 22:22:25.633737] (Utility.Process) process [72349] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","rev-parse","--verify","--quiet","59ceb4035e85ed634375596e4a44c5b3ad300652:"]
+[2021-12-06 22:22:25.637618] (Utility.Process) process [72349] done ExitSuccess
+[2021-12-06 22:22:25.63843] (Utility.Process) process [72350] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","mktree","--batch","-z"]
+[2021-12-06 22:22:25.639126] (Utility.Process) process [72351] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-tree","--full-tree","-z","-t","--","ba3ecb9e4da6ac71d242696ed0d48d2fcaa45e05"]
+[2021-12-06 22:22:25.643712] (Utility.Process) process [72351] done ExitSuccess
+[2021-12-06 22:22:25.65106] (Utility.Process) process [72350] done ExitSuccess
+[2021-12-06 22:22:25.651934] (Utility.Process) process [72352] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","commit-tree","d1d9c03e2594032e2e01eec9965d978406ea910d","--no-gpg-sign","-p","59ceb4035e85ed634375596e4a44c5b3ad300652"]
+[2021-12-06 22:22:25.657279] (Utility.Process) process [72352] done ExitSuccess
+[2021-12-06 22:22:25.658145] (Utility.Process) process [72353] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","commit-tree","ba3ecb9e4da6ac71d242696ed0d48d2fcaa45e05","--no-gpg-sign","-p","e0c16268cb18fdcb965f08c1257e43de24c577dd"]
+[2021-12-06 22:22:25.663939] (Utility.Process) process [72353] done ExitSuccess
+[2021-12-06 22:22:25.664818] (Utility.Process) process [72354] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","update-ref","refs/heads/git-annex","c89b7490febb74f3224165fab88e4eb14f05b37a"]
+[2021-12-06 22:22:25.670076] (Utility.Process) process [72354] done ExitSuccess
+[2021-12-06 22:22:25.671381] (Utility.Process) process [72355] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","diff-tree","-z","--raw","--no-renames","-l0","-r","bf9aa984fda078c0f7e2ae3f416f7366a1bd6906","bf9aa984fda078c0f7e2ae3f416f7366a1bd6906","--"]
+[2021-12-06 22:22:25.67517] (Utility.Process) process [72355] done ExitSuccess
+[2021-12-06 22:22:25.67536] (Annex.Branch) read export.log
+[2021-12-06 22:22:25.675925] (Annex.Branch) set export.log
+[2021-12-06 22:22:25.676667] (Utility.Process) process [72356] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-tree","--full-tree","-z","-r","--","bf9aa984fda078c0f7e2ae3f416f7366a1bd6906"]
+[2021-12-06 22:22:25.681325] (Utility.Process) process [72357] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"]
+[2021-12-06 22:22:25.68243] (Utility.Process) process [72358] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+[2021-12-06 22:22:25.686724] (Annex.Branch) read b24/e7d/GIT--06be486d36bd2b4cb8314b7084dca8b7252f7199.log
+export film-depot .gitattributes ok
+[2021-12-06 22:22:25.687565] (Annex.Branch) read 8eb/f8d/GIT--e43b0f988953ae3a84b00331d0ccf5f7d51cb3cf.log
+export film-depot .gitignore ok
+[2021-12-06 22:22:25.688064] (Annex.Branch) read 80f/4f8/GIT--e69de29bb2d1d6434b8b29ae775ad8c2e48c5391.log
+export film-depot 00-GIT-ANNEX ok
+[2021-12-06 22:22:25.688591] (Annex.Branch) read 087/1f9/WORM-s1116285116-m1593796806--A,32Beautiful,32Mind,32,402001,41,321080p.mkv.log
+[2021-12-06 22:22:25.689105] (Annex.Branch) read b6a/0bf/WORM-s1037772729-m1595036949--Alien,32,401979,41,32720p.mkv.log
+[2021-12-06 22:22:25.689592] (Annex.Branch) read fc1/47e/WORM-s2584124634-m1593818075--Aliens,32,401986,41,32SE,321080p.mp4.log
+[2021-12-06 22:22:25.690021] (Annex.Branch) read c32/8d4/WORM-s1598749320-m1589682846--American,32Pie,32,401999,41,321080p.mp4.log
+[2021-12-06 22:22:25.69048] (Annex.Branch) read c61/d77/WORM-s2118257276-m1525912572--American,32Pie,32II,32,402001,41,32720p.mp4.log
+[2021-12-06 22:22:25.691044] (Annex.Branch) read a91/0bf/WORM-s2117183986-m1571443864--American,32Pie,32Wedding,32,402003,41,321080p.mp4.log
+[2021-12-06 22:22:25.69149] (Annex.Branch) read 6e0/e37/WORM-s2524668661-m1636382981--Armageddon,32,401998,41,321080p.mp4.log
+[2021-12-06 22:22:25.692829] (Annex.Branch) read e65/e2a/WORM-s2049268430-m1634226615--Avengers,,,32Age,32of,32Ultron,32,402015,41,321080p.mp4.log
+[2021-12-06 22:22:25.693395] (Annex.Branch) read 0dc/df0/WORM-s1175417895-m1555550689--Avengers,,,32Infinity,32War,32,402018,41.mkv.log
+[2021-12-06 22:22:25.697988] (Annex.Branch) read a08/5cc/WORM-s1073479115-m1536010249--Avengers,,,32The,32,402012,41,32720p.mkv.log
+[2021-12-06 22:22:25.698657] (Annex.Branch) read ae6/34b/WORM-s1942991012-m1595715198--Back,32to,32the,32Future,32,401985,41,321080p.mp4.log
+[2021-12-06 22:22:25.699122] (Annex.Branch) read 9a6/a47/WORM-s2108282730-m1636257924--Beverly,32Hills,32Cop,322,32,401987,41,321080p.mp4.log
+[2021-12-06 22:22:25.699818] (Annex.Branch) read 7cb/426/WORM-s1785804434-m1636253231--Beverly,32Hills,32Cop,32III,32,401994,41,321080p.mp4.log
+[2021-12-06 22:22:25.700452] (Annex.Branch) read d14/a70/WORM-s2036895920-m1595715206--Black,32Hawk,32Down,32,402001,41,321080p.mp4.log
+[2021-12-06 22:22:25.700903] (Annex.Branch) read bc1/18b/WORM-s2620361196-m1526994937--Contact,32,401997,41,321080p.mp4.log
+[2021-12-06 22:22:25.701508] (Annex.Branch) read 2de/e6b/WORM-s1937651408-m1595715213--Crimson,32Tide,32,401995,41,321080p.mp4.log
+[2021-12-06 22:22:25.70195] (Annex.Branch) read f38/990/WORM-s2211439503-m1595715222--Die,32Hard,32,401988,41,321080p.mp4.log
+[2021-12-06 22:22:25.702438] (Annex.Branch) read 95c/68d/WORM-s902398239-m1530225214--Doctor,32Strange,32,402016,41,32720p.mp4.log
+[2021-12-06 22:22:25.703064] (Annex.Branch) read 137/b62/WORM-s1768193827-m1563842861--Edge,32of,32Tomorrow,32,402014,41,321080p.mp4.log
+[2021-12-06 22:22:25.703694] (Annex.Branch) read a03/fcb/WORM-s1917114368-m1560119371--Enders,32Game,32,402013,41.avi.log
+[2021-12-06 22:22:25.7043] (Annex.Branch) read e1b/f67/WORM-s1828312956-m1586061932--Enemy,32at,32the,32Gates,32,402001,41,321080p.mkv.log
+[2021-12-06 22:22:25.704724] (Annex.Branch) read c9d/3b1/WORM-s2706201822-m1586047240--Enemy,32of,32the,32State,32,401998,41,321080p.mp4.log
+[2021-12-06 22:22:25.705143] (Annex.Branch) read 930/182/WORM-s848757174-m1518370060--Field,32of,32Dreams,32,401989,41.mp4.log
+[2021-12-06 22:22:25.705724] (Annex.Branch) read afa/30f/WORM-s2104549496-m1595715258--Fifth,32Element,,,32The,32,401997,41,321080p.mp4.log
+[2021-12-06 22:22:25.706178] (Annex.Branch) read 551/6ba/WORM-s1339997270-m1636253775--Final,32Fantasy,32The,32Spirits,32Within,32,402001,41,321080p.mkv.log
+[2021-12-06 22:22:25.706805] (Annex.Branch) read 91d/c72/WORM-s2237782219-m1613505958--First,32Knight,32,401995,41,321080p.mp4.log
+[2021-12-06 22:22:25.707349] (Annex.Branch) read ddc/080/WORM-s2637057735-m1542757748--Fugitive,,,32The,32,401993,41,32720p.mkv.log
+[2021-12-06 22:22:25.707983] (Annex.Branch) read 152/91b/WORM-s2342003785-m1518370156--Galaxy,32Quest,32,401999,41.mkv.log
+[2021-12-06 22:22:25.708543] (Annex.Branch) read e7a/984/WORM-s1756467527-m1595109447--Ghostbusters,32,401984,41,321080p.mp4.log
+[2021-12-06 22:22:25.708988] (Annex.Branch) read 0e8/ff6/WORM-s3933229071-m1602373090--Gladiator,32,402000,41,321080p.mkv.log
+[2021-12-06 22:22:25.709365] (Annex.Branch) read 87a/3d3/WORM-s1690337749-m1636382985--Groundhog,32Day,32,401993,41,321080p.mp4.log
+[2021-12-06 22:22:25.709988] (Annex.Branch) read 3bd/e70/WORM-s2046008512-m1555550686--Harry,32Brown,32,402009,41,321080p.mp4.log
+[2021-12-06 22:22:25.710588] (Annex.Branch) read 972/247/WORM-s1073833682-m1584242874--Hart,39s,32War,32,402002,41,32720p.mp4.log
+[2021-12-06 22:22:25.711033] (Annex.Branch) read a95/4ec/WORM-s790627159-m1518370226--Hunt,32for,32Red,32October,,,32The.mkv.log
+[2021-12-06 22:22:25.711609] (Annex.Branch) read d45/158/WORM-s1048990304-m1636829207--In,32the,32Heat,32of,32the,32Night,32,401967,41,32720p.mkv.log
+[2021-12-06 22:22:25.712203] (Annex.Branch) read 0f6/7f5/WORM-s2479265386-m1595715232--Inception,32,402010,41,321080p.mp4.log
+[2021-12-06 22:22:25.712624] (Annex.Branch) read b9f/faa/WORM-s1855071446-m1525796911--Independence,32Day,32,401996,41.mp4.log
+[2021-12-06 22:22:25.713269] (Annex.Branch) read bb9/0bf/WORM-s2123742199-m1595715241--Indiana,32Jones,32and,32the,32L-86e667a21adfdf59df66cde8e3c653de.log
+[2021-12-06 22:22:25.713841] (Annex.Branch) read 7d3/b92/WORM-s1832093728-m1634098515--Indiana,32Jones,32and,32the,32Raiders,32of,32the,32Lost,32Ark,32,401981,41,321080p.mp4.log
+[2021-12-06 22:22:25.714393] (Annex.Branch) read 7fe/c79/WORM-s3384564503-m1636826013--Inglourious,32Basterds,32,402009,41,321080p.mkv.log
+[2021-12-06 22:22:25.714964] (Annex.Branch) read 1b1/94b/WORM-s2640394877-m1598221200--Insider,,,32The,32,401999,41,321080p.mp4.log
+[2021-12-06 22:22:25.715363] (Annex.Branch) read 496/2db/WORM-s2431697820-m1636258658--Interstellar,32,402014,41,321080p.mp4.log
+[2021-12-06 22:22:25.716042] (Annex.Branch) read ec2/268/WORM-s1282009570-m1518370381--Iron,32Man.mkv.log
+[2021-12-06 22:22:25.716841] (Annex.Branch) read d9c/575/WORM-s2280602670-m1596874458--Kelly,39s,32Heroes,32,401970,41,321080p.mp4.log
+[2021-12-06 22:22:25.7175] (Annex.Branch) read fb8/a6a/WORM-s1532588342-m1518370430--King,32Author.mkv.log
+[2021-12-06 22:22:25.71824] (Annex.Branch) read 1e8/0c4/WORM-s2362174469-m1596874396--L.A.,32Confidential,32,401997,41,321080p.mp4.log
+[2021-12-06 22:22:25.718719] (Annex.Branch) read bdf/208/WORM-s2580319882-m1597519936--Last,32Samurai,,,32The,32,402003,41,321080p.mp4.log
+[2021-12-06 22:22:25.719159] (Annex.Branch) read c83/4b8/WORM-s2200470144-m1518370504--Last,32of,32the,32Mohicans,,,32-3e3f78ef1cecfd2d87dd70a706077bfb.log
+[2021-12-06 22:22:25.719718] (Annex.Branch) read 7ca/016/WORM-s1615545853-m1597087605--Lethal,32Weapon,32,401987,41,321080p.mp4.log
+[2021-12-06 22:22:25.720106] (Annex.Branch) read fbe/3b4/WORM-s1041661875-m1636833567--Lethal,32Weapon,322,32,401989,41,32720p.mkv.log
+[2021-12-06 22:22:25.720762] (Annex.Branch) read bfb/f3d/WORM-s832138999-m1536079832--Letters,32From,32Iwo,32Jima,32,402006,41,32720p.mkv.log
+[2021-12-06 22:22:25.721395] (Annex.Branch) read e18/c6d/WORM-s2246486055-m1523923497--Lord,32Of,32The,32Rings,,,32The-8d99f15c7937db1ce3c48c84d452f106.log
+[2021-12-06 22:22:25.721973] (Annex.Branch) read d9e/f96/WORM-s3806431573-m1524068665--Lord,32of,32the,32Rings,,,32The-a01e8f7ccb49401f72481cac93bbce95.log
+[2021-12-06 22:22:25.722584] (Annex.Branch) read bfa/477/WORM-s2248342870-m1524067831--Lord,32Of,32The,32Rings,,,32The-dddba4061a1be729c40b94630b22efb3.log
+[2021-12-06 22:22:25.723152] (Annex.Branch) read 911/69b/WORM-s2173429952-m1597087612--Manchurian,32Candidate,,,32The,32,402004,41,321080p.mp4.log
+[2021-12-06 22:22:25.723603] (Annex.Branch) read ab2/50d/WORM-s2538513668-m1518370740--Mask,32of,32Zorro,,,32The.mp4.log
+[2021-12-06 22:22:25.724229] (Annex.Branch) read ab8/84c/WORM-s1990201957-m1563568759--Oblivion,32,402013,41,321080p.mp4.log
+[2021-12-06 22:22:25.724791] (Annex.Branch) read fe2/13e/WORM-s865388563-m1518370822--Office,32Space.mkv.log
+[2021-12-06 22:22:25.725374] (Annex.Branch) read b86/aa5/WORM-s2144329193-m1636914388--Pirates,32of,32the,32Caribbean,-e8703135133e2788f38446549a5159ef.log
+[2021-12-06 22:22:25.725957] (Annex.Branch) read efa/8d7/WORM-s1185325919-m1518868254--Postman,,,32The,32,401997,41.mp4.log
+[2021-12-06 22:22:25.72656] (Annex.Branch) read 87d/859/WORM-s2059690008-m1585382324--Prestige,,,32The,32,402006,41,321080p.mkv.log
+[2021-12-06 22:22:25.726965] (Annex.Branch) read 41e/61a/GIT--9ec57d0596ccb70cfd6255cb85d89d00fce5a43a.log
+export film-depot README.md ok
+[2021-12-06 22:22:25.727519] (Annex.Branch) read 68f/38d/WORM-s948100504-m1574090603--Regarding,32Henry,32,401991,41,32720p.mp4.log
+[2021-12-06 22:22:25.727919] (Annex.Branch) read 817/23b/WORM-s878672153-m1574033361--Risky,32Business,32,401983,41,32720p.mp4.log
+[2021-12-06 22:22:25.72833] (Annex.Branch) read 1da/bbf/WORM-s1231862861-m1518370935--Rock,,,32The.mkv.log
+[2021-12-06 22:22:25.728893] (Annex.Branch) read f32/87e/WORM-s1816321857-m1598221210--Rocketeer,,,32The,32,401991,41,321080p.mp4.log
+[2021-12-06 22:22:25.729545] (Annex.Branch) read f5c/cbf/WORM-s2787891812-m1630883712--Rogue,32One,32A,32Star,32Wars,32Story,32,402016,41,321080p.mkv.log
+[2021-12-06 22:22:25.730174] (Annex.Branch) read dc3/69b/WORM-s1544737178-m1524572776--Serenity,32,402005,41,32720p.mp4.log
+[2021-12-06 22:22:25.730906] (Annex.Branch) read 831/8c8/WORM-s2384387600-m1598221224--Shawshank,32Redemption,,,32The,32,401994,41,321080p.mp4.log
+[2021-12-06 22:22:25.731345] (Annex.Branch) read e8f/5db/WORM-s1979737445-m1584227786--Sneakers,32,401992,41,321080p.mp4.log
+[2021-12-06 22:22:25.732205] (Annex.Branch) read 234/d7a/WORM-s3665537220-m1634232478--Star,32Trek,32II,32The,32Wrath,32of,32Khan,32,401982,41,321080p.mkv.log
+[2021-12-06 22:22:25.733026] (Annex.Branch) read cc6/1f9/WORM-s1853412510-m1638068017--Star,32Trek,32III,32The,32Search,32for,32Spock,32,401984,41,321080p.mkv.log
+[2021-12-06 22:22:25.733609] (Annex.Branch) read 23e/96e/WORM-s2133269783-m1638593594--Star,32Trek,32IV,32The,32Voyage,32Home,32,401986,41,321080p.mkv.log
+[2021-12-06 22:22:25.734223] (Annex.Branch) read 95e/21c/WORM-s2313033711-m1638593604--Star,32Trek,32The,32Motion,32Picture,32,401979,41,321080p.mkv.log
+[2021-12-06 22:22:25.734777] (Annex.Branch) read 998/5e9/WORM-s1632505949-m1638593600--Star,32Trek,32V,32The,32Final,32Frontier,32,401989,41,321080p.mkv.log
+[2021-12-06 22:22:25.735323] (Annex.Branch) read 57c/f2d/WORM-s1725216326-m1638593601--Star,32Trek,32VI,32The,32Undiscovered,32Country,32,401991,41,321080p.mkv.log
+[2021-12-06 22:22:25.735905] (Annex.Branch) read 786/e15/WORM-s2297779450-m1603679910--Stargate,32,401994,41,32720p.mkv.log
+[2021-12-06 22:22:25.73647] (Annex.Branch) read 393/ea3/WORM-s2168164654-m1601834448--Starship,32Troopers,32,401997,41,321080p.mp4.log
+[2021-12-06 22:22:25.736868] (Annex.Branch) read 532/60a/WORM-s2312036493-m1582511386--Terminator,322,,,32Judgment,32Day,32,401991,41,321080p.mkv.log
+[2021-12-06 22:22:25.737335] (Annex.Branch) read d47/eb9/WORM-s1824450643-m1601834456--Terminator,323,,,32Rise,32of,32-84776e11fa5dc1fb1fecfe410230caa5.log
+[2021-12-06 22:22:25.737736] (Annex.Branch) read c74/c2a/WORM-s1499960820-m1636255738--Terminator,32Genisys,32,402015,41,321080p.mp4.log
+[2021-12-06 22:22:25.738318] (Annex.Branch) read 31c/59e/WORM-s1426550835-m1599521599--Terminator,,,32The,32,401984,41,321080p.mkv.log
+[2021-12-06 22:22:25.738713] (Annex.Branch) read d81/d11/WORM-s2506262160-m1637273233--The,32Matrix,32,401999,41,321080p.mp4.log
+[2021-12-06 22:22:25.739251] (Annex.Branch) read 531/658/WORM-s1203158836-m1518370785--Mummy,,,32The.mkv.log
+[2021-12-06 22:22:25.73977] (Annex.Branch) read 0b3/5e4/WORM-s1945784875-m1518370865--Patriot,,,32The.mkv.log
+[2021-12-06 22:22:25.740366] (Annex.Branch) read 0f3/e1c/WORM-s1938608286-m1599511452--Three,32Kings,32,401999,41,321080p.mp4.log
+[2021-12-06 22:22:25.74077] (Annex.Branch) read f38/7ec/WORM-s1463832198-m1518371391--Three,32Musketeers,,,32The,32,401993,41.avi.log
+[2021-12-06 22:22:25.741289] (Annex.Branch) read b7b/e4c/WORM-s2041175346-m1601834465--Training,32Day,32,402001,41,321080p.mp4.log
+[2021-12-06 22:22:25.741684] (Annex.Branch) read 289/8d3/WORM-s2094454430-m1601834475--Tron,32Legacy,32,402010,41,321080p.mp4.log
+[2021-12-06 22:22:25.742067] (Annex.Branch) read 960/971/WORM-s5853485850-m1638199114--Underworld,32,402003,41,321080p.mkv.log
+[2021-12-06 22:22:25.742606] (Annex.Branch) read af1/ce9/WORM-s1720391416-m1586034793--Untouchables,,,32The,32,401987,41,321080p.mkv.log
+[2021-12-06 22:22:25.743017] (Annex.Branch) read 239/3b9/WORM-s1827306623-m1563582252--Valkyrie,32,402008,41,321080p.mp4.log
+[2021-12-06 22:22:25.743606] (Annex.Branch) read 148/196/WORM-s1836438038-m1584843373--WarGames,32,401983,41,321080p.mp4.log
+[2021-12-06 22:22:25.743988] (Annex.Branch) read a18/b70/WORM-s1481063951-m1518371518--Watchmen,,,32The.mkv.log
+[2021-12-06 22:22:25.744522] (Annex.Branch) read 01c/42b/WORM-s2648951821-m1584890677--Where,32Eagles,32Dare,32,401968,41,321080p.mp4.log
+[2021-12-06 22:22:25.744909] (Annex.Branch) read 531/f42/WORM-s2173033800-m1584848357--Willow,32,401988,41,321080p.mp4.log
+[2021-12-06 22:22:25.745316] (Annex.Branch) read e2b/244/WORM-s1151346022-m1574909969--Witness,32,401985,41,32720p.mkv.log
+[2021-12-06 22:22:25.745758] (Annex.Branch) read 044/bea/WORM-s1077510790-m1476055956--X-Men,,,32Apocalypse,32,402016,41.mkv.log
+[2021-12-06 22:22:25.7463] (Annex.Branch) read 04d/db6/WORM-s1618758250-m1520576862--X-Men,,,32Days,32Of,32Future,32Past,32,402014,41.mkv.log
+[2021-12-06 22:22:25.746896] (Annex.Branch) read 1e7/136/WORM-s1656445231-m1530398504--X-Men,,,32Last,32Stand,,,32The,32,402006,41,32720p.mkv.log
+[2021-12-06 22:22:25.74734] (Utility.Process) process [72356] done ExitSuccess
+[2021-12-06 22:22:25.74821] (Utility.Process) process [72359] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/remotes/film-depot/master"]
+[2021-12-06 22:22:25.75349] (Utility.Process) process [72359] done ExitSuccess
+[2021-12-06 22:22:25.754349] (Utility.Process) process [72360] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/remotes/film-depot/master","--full-history","--no-abbrev","--format=%T %H %P"]
+[2021-12-06 22:22:25.758299] (Utility.Process) process [72360] done ExitFailure (-13)
+[2021-12-06 22:22:25.759069] (Utility.Process) process [72361] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","update-ref","refs/remotes/film-depot/master","9d18fec41a08dbcd0377a792d987be7c158fedbc"]
+[2021-12-06 22:22:25.762365] (Utility.Process) process [72361] done ExitSuccess
+[2021-12-06 22:22:25.769741] (Utility.Process) process [72362] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","hash-object","-w","--stdin-paths","--no-filters"]
+[2021-12-06 22:22:25.770564] (Utility.Process) process [72363] feed: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","update-index","-z","--index-info"]
+[2021-12-06 22:22:25.777266] (Utility.Process) process [72363] done ExitSuccess
+[2021-12-06 22:22:25.778167] (Utility.Process) process [72364] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2021-12-06 22:22:25.783594] (Utility.Process) process [72364] done ExitSuccess
+(recording state in git...)
+[2021-12-06 22:22:25.78468] (Utility.Process) process [72365] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","write-tree"]
+[2021-12-06 22:22:25.791959] (Utility.Process) process [72365] done ExitSuccess
+[2021-12-06 22:22:25.792898] (Utility.Process) process [72366] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","commit-tree","e5e18bf030c0e281b800e7b033c6132564bfff31","--no-gpg-sign","-p","refs/heads/git-annex"]
+[2021-12-06 22:22:25.798507] (Utility.Process) process [72366] done ExitSuccess
+[2021-12-06 22:22:25.79938] (Utility.Process) process [72367] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","update-ref","refs/heads/git-annex","429f678d58abe674707f99a1d45d586c125fb3cd"]
+[2021-12-06 22:22:25.804332] (Utility.Process) process [72367] done ExitSuccess
+[2021-12-06 22:22:25.805644] (Utility.Process) process [72357] done ExitSuccess
+[2021-12-06 22:22:25.805969] (Utility.Process) process [72358] done ExitSuccess
+[2021-12-06 22:22:25.806327] (Utility.Process) process [72341] done ExitSuccess

(Diff truncated)

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
index f809296447..6aad55c1ed 100644
--- a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
@@ -58,3 +58,5 @@ per my setting in `~/.gitconfig` (which also has `annex.retry = 3`)
 
 
 [[!meta title="annex.pidlock can cause git-annex get -J to fail to take transfer lock"]]
+
+> [[done]] --[[Joey]] 

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment
index 6eb6b9e571..1011eaccd3 100644
--- a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment
@@ -5,12 +5,4 @@
  content="""
 Update: All problems noted above are fixed, fine-grained locking is merged, 
 and it seems to work well!
-
-Except for this, which I have not gotten to. It may not actually matter, but
-worth looking at to be sure: 
-
-`getLockStatus` and `checkSaneLock` look at the status of the pid lock,
-but not yet at the fine-grained STM lock status. And as implemented,
-I think they never worked at all, since they check for posix locks on the
-pid lock file.
 """]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment
new file mode 100644
index 0000000000..6eb6b9e571
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_10_1cbef11391d23210230872a75f8e5414._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2021-12-06T19:11:20Z"
+ content="""
+Update: All problems noted above are fixed, fine-grained locking is merged, 
+and it seems to work well!
+
+Except for this, which I have not gotten to. It may not actually matter, but
+worth looking at to be sure: 
+
+`getLockStatus` and `checkSaneLock` look at the status of the pid lock,
+but not yet at the fine-grained STM lock status. And as implemented,
+I think they never worked at all, since they check for posix locks on the
+pid lock file.
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index cfb5f6092c..69d2da9f24 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,7 +6,6 @@ git-annex (8.20211124) UNRELEASED; urgency=medium
     erroring out.
   * export: Avoid unncessarily re-exporting non-annexed files that were
     already exported.
-  * Fix locking bug introduced in version 8.20200814.
   * Fix locking problems when annex.pidlock is set and concurrency is
     enabled eg with -J.
 
diff --git a/Utility/LockPool/STM.hs b/Utility/LockPool/STM.hs
index d5ab6b4e41..0d54d3dd06 100644
--- a/Utility/LockPool/STM.hs
+++ b/Utility/LockPool/STM.hs
@@ -28,7 +28,6 @@ import System.FilePath.ByteString (RawFilePath)
 import qualified Data.Map.Strict as M
 import Control.Concurrent.STM
 import Control.Exception
-import Control.Monad
 
 type LockFile = RawFilePath
 
@@ -129,8 +128,8 @@ getLockStatus pool file getdefault checker = do
 		Nothing -> getdefault
 		Just restore -> bracket_ (return ()) restore checker
 
--- Only runs action to close underlying lock file when this is the last
--- user of the lock, and when the lock has not already been closed.
+-- Releases the lock. When it is a shared lock, it may remain locked by
+-- other LockHandles.
 --
 -- Note that the lock pool is left empty while the CloseLockFile action
 -- is run, to avoid race with another thread trying to open the same lock
@@ -139,15 +138,15 @@ releaseLock :: LockHandle -> IO ()
 releaseLock h = go =<< atomically (tryTakeTMVar h)
   where
 	go (Just (pool, file, closelockfile)) = do
-		(m, lastuser) <- atomically $ do
+		m <- atomically $ do
 			m <- takeTMVar pool
 			return $ case M.lookup file m of
 				Just (LockStatus mode n firstlocksem)
-					| n == 1 -> (M.delete file m, True)
+					| n == 1 -> (M.delete file m)
 					| otherwise ->
-						(M.insert file (LockStatus mode (pred n) firstlocksem) m, False)
-				Nothing -> (m, True)
-		() <- when lastuser closelockfile
+						(M.insert file (LockStatus mode (pred n) firstlocksem) m)
+				Nothing -> m
+		() <- closelockfile
 		atomically $ putTMVar pool m
 	-- The LockHandle was already closed.
 	go Nothing = return ()
diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment
new file mode 100644
index 0000000000..cba3c4624c
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_9_df98c47016e79f15218880a009e671ea._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2021-12-06T16:36:45Z"
+ content="""
+Unfortunately, [[!commit 66b2536ea0aa5c88f5b744eeace3322a8a4a10b6]]
+broke shared locking when pid locks are not used, causing a FD leak,
+and had to be reverted. Which further breaks concurrent pid locking. 
+
+At this point, when -J is used with pidlock, it may drop the lock but
+still think it has it held internally. Comment #7 found another way that
+could happen, and the conclusion is that the pid lock must migrate away
+from using the lock pool.
+"""]]

diff --git a/doc/forum/conflict_with_git-crypt.mdwn b/doc/forum/conflict_with_git-crypt.mdwn
new file mode 100644
index 0000000000..0484ea5c87
--- /dev/null
+++ b/doc/forum/conflict_with_git-crypt.mdwn
@@ -0,0 +1,3 @@
+[git-crypt](https://www.agwa.name/projects/git-crypt/) is executed as a filter for secrets specified in `.gitattributes` files. However, git-annex initialization results in `* filter=annex` being added to `$GIT_DIR/info/attributes`, which [has the highest precedence](https://www.git-scm.com/docs/gitattributes#_description), ultimately causing any git-crypt filtering to be bypassed.
+
+Is there a recommended workaround for this? Thanks!

diff --git a/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn
new file mode 100644
index 0000000000..db4efbbe1b
--- /dev/null
+++ b/doc/forum/Use_assistant_to_sync_just_git_repos__63__.mdwn
@@ -0,0 +1,11 @@
+Hi,
+
+So lately I've been using git-annex to sync repos with only markdown files for Obsidian. Running `git annex sync --no-content` works well for this. But I have to remember when I've updated my Obsidian files on one system and make sure to sync to other repos.
+
+Could I use the assistant for this? For repos with annexed files, I've always preferred to manually manage my repos, so I've never used the assistant.
+
+Ideally, as I commit changes, it tries to sync them to other systems, and if those systems are offline (my MBP is closed for example and the ssh connection therefore times out), nothing explodes.
+
+Is this possible? Is it possible with a half dozen similar repos? Can a single agent do this or do I need to start one for each repo?
+
+Thanks!

diff --git a/doc/thanks/list b/doc/thanks/list
index 1876e7f234..9a57d462c5 100644
--- a/doc/thanks/list
+++ b/doc/thanks/list
@@ -121,3 +121,4 @@ Markus,
 Nick, 
 Nicholas Golder-Manning, 
 Troels Henriksen, 
+Max Thoursie, 

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment
new file mode 100644
index 0000000000..36bc79daa6
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_d8ccfe3fe0d5123c1065ace73b623b44._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2021-12-03T22:35:16Z"
+ content="""
+Unfortunately the more I dig into pid locking, the more deep problems I'm
+finding..
+
+When the pid lock's LockHandle is dropped, it drops the pid lock, and that
+happens even if there is another LockHandle for the pid lock, which is
+possble since it's a shared lock. So the pid lock may go away despite
+a thread is still operating as if it is present. I think probably the pid
+lock needs to stop using the LockPool and use a single LockHandle, which is
+ref counted.
+
+`getLockStatus` and `checkSaneLock` look at the status of the pid lock,
+but not yet at the fine-grained STM lock status. And as implemented,
+I think they never worked at all, since they check for posix locks on the
+pid lock file.
+"""]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment
new file mode 100644
index 0000000000..fbdadc2375
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_7_51f6745c4abce7b3cdfcf1c5998f8329._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 7"
+ date="2021-12-03T21:41:53Z"
+ content="""
+Great - thank you! FWIW I gave `8.20211123+git32-g9c0f3d1de-1~ndall+1_amd64` build a test run on that system, and it performed nicely (did get all files in my limited `get -J5` attempt on that dataset).  Will be waiting for the ultimate solution to try.
+"""]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_6_2c83482ead69a902c99fca3ae63dfc34._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_6_2c83482ead69a902c99fca3ae63dfc34._comment
new file mode 100644
index 0000000000..9138ef0207
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_6_2c83482ead69a902c99fca3ae63dfc34._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2021-12-03T20:37:18Z"
+ content="""
+I noticed that there is a FD leak when pidlock is enabled.
+About 1 file in 10 that is processed,
+there is a handle to ".git/annex/pidlock.lck" that is left open.
+This is not a regression caused by my fix above, bug is in older git-annex
+too.
+
+Also, I have the fine-grained locking implemented in the pidlockfinegrained
+branch now, but am waiting to merge it for testing and to resolve the above
+problem first.
+"""]]

diff --git a/Utility/LockPool/Windows.hs b/Utility/LockPool/Windows.hs
index 9f3a0b95cc..4b35da6b0f 100644
--- a/Utility/LockPool/Windows.hs
+++ b/Utility/LockPool/Windows.hs
@@ -24,7 +24,7 @@ import Utility.LockPool.STM (LockFile, LockMode(..))
 lockShared :: LockFile -> IO (Maybe LockHandle)
 lockShared file = tryMakeLockHandle P.lockPool file
 	(\p f -> P.tryTakeLock p f LockShared)
-	(\f -> fmap mk <$> F.lockShared f)
+	(\f _ -> fmap mk <$> F.lockShared f)
 
 {- Tries to take an exclusive lock on a file. Fails if another process has
  - a shared or exclusive lock.
@@ -35,7 +35,7 @@ lockShared file = tryMakeLockHandle P.lockPool file
 lockExclusive :: LockFile -> IO (Maybe LockHandle)
 lockExclusive file = tryMakeLockHandle P.lockPool file
 	(\p f -> P.tryTakeLock p f LockExclusive)
-	(\f -> fmap mk <$> F.lockExclusive f)
+	(\f _ -> fmap mk <$> F.lockExclusive f)
 
 {- If the initial lock fails, this is a BUSY wait, and does not
  - guarentee FIFO order of waiters. In other news, Windows is a POS. -}
diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_5_3b840d50a1ecc9cb6e161e3c039fe332._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_5_3b840d50a1ecc9cb6e161e3c039fe332._comment
new file mode 100644
index 0000000000..264dfcfa3c
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_5_3b840d50a1ecc9cb6e161e3c039fe332._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2021-12-03T18:07:56Z"
+ content="""
+Indeed, fixed that.
+"""]]

diff --git a/doc/forum/Sharing_over_SMB__47__NFS_while_running_assistant__63__.mdwn b/doc/forum/Sharing_over_SMB__47__NFS_while_running_assistant__63__.mdwn
new file mode 100644
index 0000000000..365f30dc22
--- /dev/null
+++ b/doc/forum/Sharing_over_SMB__47__NFS_while_running_assistant__63__.mdwn
@@ -0,0 +1,15 @@
+I have a large collection of media files (music, tv shows, movies, random things I've recorded from television and converted from VHS, random videos from the internet that I've saved) that are stored safely on my TrueNAS box. I've been toying with the idea of importing them into git-annex - I am already using annex for other things, and it's great. :-)
+
+
+Currently, the TrueNAS machine shares these files out over the network as Samba and NFS shares. There's a couple of subdirs that get modified frequently by the likes of Sonarr and Radarr downloading stuff from Usenet - these are running on a Windows machine, which has the shares mounted via Samba. I would like to have git-annex deal with these files, but I have concerns around how (a) the shares would work, (b) how to deal with autonomous file changes from the likes of Radarr and (c) any potential corruption issues stemming from the two.
+
+I did a small scale test where I created a jail on my TrueNAS box, mounted the shared drive and created a test subdirectory and filled it with 40GBs of exemplar data. After importing this data into annex, I found that running annex assistant in the jail and modifying the directory (over the Samba share from the machine that runs Sonarr/Radarr) seemed to work well enough - in that there was a commit corresponding to whatever changed without me intervening. This seems ok, and even viewing the media over the Samba share on a Windows machine (to my surprise) worked fine - it didn't even care or seem to notice that the files were now technically stored as symlinks.
+
+I was happy to see this largely go ok, so my plan is (right now) to simply create a couple of different annexes using the existing file trees and continue as normal (no changes to how shares work, or to Sonarr or Radarr). To deal with Sonarr and Radarr's changes to the files, I'd run an annex assistant in the jail for each of the folders to make the commits automatically as the machine makes changes to files over the Samba share.
+
+Is this a terrible idea? What sort of issues am I likely to have?
+
+
+
+
+

diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_705616c61fe7a0779df3794fa8a1d6dc._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_705616c61fe7a0779df3794fa8a1d6dc._comment
new file mode 100644
index 0000000000..1db847262d
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_705616c61fe7a0779df3794fa8a1d6dc._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2021-12-02T20:42:37Z"
+ content="""
+backref: follow up is now filed separately [here](https://git-annex.branchable.com/bugs/merge-annex-branches__61__false_-_automate_and_extend/)
+"""]]

diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment
deleted file mode 100644
index 8f31de3176..0000000000
--- a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment
+++ /dev/null
@@ -1,22 +0,0 @@
-[[!comment format=mdwn
- username="yarikoptic"
- avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
- subject="automate + extend"
- date="2021-12-02T20:34:55Z"
- content="""
-- is there a chance for that option/mode of operation to get auto enabled whenever encountering \"read-only\" situation?  could inform user on stderr that no merge is happening etc
-- could other \"read only\" commands be fixed up to allow for such a mode of operation?  I tried on 'wanted' and it didn't help fully:
-
-```
-[d31548v@discovery7 1076_spacetop]$ git annex wanted here
-git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
-git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
-[d31548v@discovery7 1076_spacetop]$ git  -c annex.merge-annex-branches=false annex wanted here
-
-git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
-[d31548v@discovery7 1076_spacetop]$ git annex version | head -n 1
-git-annex version: 8.20211117+git14-ge1f38b9dd-1~ndall+1
-
-```
-
-"""]]

diff --git a/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend.mdwn b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend.mdwn
new file mode 100644
index 0000000000..fd3f8cc74c
--- /dev/null
+++ b/doc/bugs/merge-annex-branches__61__false_-_automate_and_extend.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+
+previously reported and partially addressed in [here](https://git-annex.branchable.com/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/)
+
+
+### What steps will reproduce the problem?
+
+try any presumably read-only annex operation (whereis, wanted, etc) on a repository where you have no write permissions.
+
+` -c annex.merge-annex-branches=false ` configuration switch was introduced to allow for running `whereis` in "read-only" mode.  But it requires user knowhow and good memory.
+It is also inconsistent with how `git` behaves in that it doesn't require any extra option and just handles it automagically/gracefully.
+
+If git-annex could automagically switch to such mode when encountering read-only repository, may be announcing that on stderr, that would make life so much easier.
+
+Also it seems that some commands are not even fully support it:
+
+```
+[d31548v@discovery7 1076_spacetop]$ git annex wanted here
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+[d31548v@discovery7 1076_spacetop]$ git  -c annex.merge-annex-branches=false annex wanted here
+
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+[d31548v@discovery7 1076_spacetop]$ git annex version | head -n 1
+git-annex version: 8.20211117+git14-ge1f38b9dd-1~ndall+1
+```
+
+[[!meta author=yoh]]
+[[!tag projects/datalad]]

diff --git a/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment
new file mode 100644
index 0000000000..8f31de3176
--- /dev/null
+++ b/doc/bugs/impossible_to_perform___34__read-only__34___git_annex_info_without_write_permissions/comment_4_72d975f81eb3aee600ea7a02e2f8855a._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="automate + extend"
+ date="2021-12-02T20:34:55Z"
+ content="""
+- is there a chance for that option/mode of operation to get auto enabled whenever encountering \"read-only\" situation?  could inform user on stderr that no merge is happening etc
+- could other \"read only\" commands be fixed up to allow for such a mode of operation?  I tried on 'wanted' and it didn't help fully:
+
+```
+[d31548v@discovery7 1076_spacetop]$ git annex wanted here
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+[d31548v@discovery7 1076_spacetop]$ git  -c annex.merge-annex-branches=false annex wanted here
+
+git-annex: /dartfs/rc/lab/D/DBIC/DBIC/archive/BIDS/Wager/Wager/1076_spacetop/.git/annex: openTempFile template locktmp: permission denied (Permission denied)
+[d31548v@discovery7 1076_spacetop]$ git annex version | head -n 1
+git-annex version: 8.20211117+git14-ge1f38b9dd-1~ndall+1
+
+```
+
+"""]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_4_089f1d2d7de435a98dc1098ebf1cf3df._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_4_089f1d2d7de435a98dc1098ebf1cf3df._comment
new file mode 100644
index 0000000000..cd1b70d472
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_4_089f1d2d7de435a98dc1098ebf1cf3df._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2021-12-02T13:05:55Z"
+ content="""
+Just a note that I believe locking changes broke windows build, can see log [here](https://github.com/datalad/git-annex/runs/4390439197?check_suite_focus=true#step:18:205)
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index 4a7d78da3a..cfb5f6092c 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -7,6 +7,8 @@ git-annex (8.20211124) UNRELEASED; urgency=medium
   * export: Avoid unncessarily re-exporting non-annexed files that were
     already exported.
   * Fix locking bug introduced in version 8.20200814.
+  * Fix locking problems when annex.pidlock is set and concurrency is
+    enabled eg with -J.
 
  -- Joey Hess <id@joeyh.name>  Tue, 23 Nov 2021 15:58:27 -0400
 
diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs
index 4929c42d56..786b961572 100644
--- a/Utility/LockFile/PidLock.hs
+++ b/Utility/LockFile/PidLock.hs
@@ -1,6 +1,6 @@
 {- pid-based lock files
  -
- - Copyright 2015-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2021 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -11,6 +11,8 @@ module Utility.LockFile.PidLock (
 	LockHandle,
 	tryLock,
 	waitLock,
+	waitedLock,
+	alreadyLocked,
 	dropLock,
 	LockStatus(..),
 	getLockStatus,
@@ -149,8 +151,8 @@ tryLock lockfile = do
 		setFileMode tmp' (combineModes readModes)
 		hPutStr h . show =<< mkPidLock
 		hClose h
-		let failedlock st = do
-			dropLock $ LockHandle tmp' st sidelock
+		let failedlock = do
+			dropSideLock sidelock
 			removeWhenExistsWith removeLink tmp'
 			return Nothing
 		let tooklock st = return $ Just $ LockHandle abslockfile st sidelock
@@ -171,7 +173,7 @@ tryLock lockfile = do
 						-- stale, and can take it over.
 						rename tmp' abslockfile
 						tooklock tmpst
-					_ -> failedlock tmpst
+					_ -> failedlock
 
 -- Linux's open(2) man page recommends linking a pid lock into place,
 -- as the most portable atomic operation that will fail if
@@ -254,8 +256,8 @@ checkInsaneLustre dest = do
 --
 -- After the first second waiting, runs the callback to display a message,
 -- so the user knows why it's stalled.
-waitLock :: MonadIO m => Seconds -> LockFile -> (String -> m ()) -> m LockHandle
-waitLock (Seconds timeout) lockfile displaymessage = go timeout
+waitLock :: MonadIO m => Seconds -> LockFile -> (String -> m ()) -> (Bool -> IO ()) -> m LockHandle
+waitLock (Seconds timeout) lockfile displaymessage sem = go timeout
   where
 	go n
 		| n > 0 = liftIO (tryLock lockfile) >>= \case
@@ -264,10 +266,25 @@ waitLock (Seconds timeout) lockfile displaymessage = go timeout
 					displaymessage $ "waiting for pid lock file " ++ fromRawFilePath lockfile ++ " which is held by another process (or may be stale)"
 				liftIO $ threadDelaySeconds (Seconds 1)
 				go (pred n)
-			Just lckh -> return lckh
+			Just lckh -> do
+				liftIO $ sem True
+				return lckh
 		| otherwise = do
-			displaymessage $ show timeout ++ " second timeout exceeded while waiting for pid lock file " ++ fromRawFilePath lockfile
-			giveup $ "Gave up waiting for pid lock file " ++ fromRawFilePath lockfile
+			liftIO $ sem False
+			waitedLock (Seconds timeout) lockfile displaymessage
+
+waitedLock :: MonadIO m => Seconds -> LockFile -> (String -> m ()) -> m LockHandle
+waitedLock (Seconds timeout) lockfile displaymessage = do
+	displaymessage $ show timeout ++ " second timeout exceeded while waiting for pid lock file " ++ fromRawFilePath lockfile
+	giveup $ "Gave up waiting for pid lock file " ++ fromRawFilePath lockfile
+
+-- | Use when the pid lock has already been taken by another thread of the
+-- same process, or perhaps is in the process of being taken.
+alreadyLocked :: MonadIO m => LockFile -> m LockHandle
+alreadyLocked lockfile = liftIO $ do
+	abslockfile <- absPath lockfile
+	st <- getFileStatus abslockfile
+	return $ LockHandle abslockfile st Nothing
 
 dropLock :: LockHandle -> IO ()
 dropLock (LockHandle lockfile _ sidelock) = do
diff --git a/Utility/LockPool/LockHandle.hs b/Utility/LockPool/LockHandle.hs
index e2a954fdeb..cbc649c803 100644
--- a/Utility/LockPool/LockHandle.hs
+++ b/Utility/LockPool/LockHandle.hs
@@ -52,33 +52,33 @@ makeLockHandle
 	:: (MonadIO m, MonadMask m)
 	=> P.LockPool
 	-> LockFile
-	-> (P.LockPool -> LockFile -> STM P.LockHandle)
-	-> (LockFile -> m FileLockOps)
+	-> (P.LockPool -> LockFile -> STM (P.LockHandle, P.FirstLock))
+	-> (LockFile -> P.FirstLock -> m FileLockOps)
 	-> m LockHandle
 makeLockHandle pool file pa fa = bracketOnError setup cleanup go
   where
 	setup = debugLocks $ liftIO $ atomically (pa pool file)
-	cleanup ph = debugLocks $ liftIO $ P.releaseLock ph
-	go ph = liftIO . mkLockHandle ph =<< fa file
+	cleanup (ph, _) = debugLocks $ liftIO $ P.releaseLock ph
+	go (ph, firstlock) = liftIO . mkLockHandle ph =<< fa file firstlock
 
 tryMakeLockHandle
 	:: (MonadIO m, MonadMask m)
 	=> P.LockPool
 	-> LockFile
-	-> (P.LockPool -> LockFile -> STM (Maybe P.LockHandle))
-	-> (LockFile -> m (Maybe FileLockOps))
+	-> (P.LockPool -> LockFile -> STM (Maybe (P.LockHandle, P.FirstLock)))
+	-> (LockFile -> P.FirstLock -> m (Maybe FileLockOps))
 	-> m (Maybe LockHandle)
 tryMakeLockHandle pool file pa fa = bracketOnError setup cleanup go
   where
 	setup = liftIO $ atomically (pa pool file)
 	cleanup Nothing = return ()
-	cleanup (Just ph) = liftIO $ P.releaseLock ph
+	cleanup (Just (ph, _)) = liftIO $ P.releaseLock ph
 	go Nothing = return Nothing
-	go (Just ph) = do
-		mfo <- fa file
+	go (Just (ph, firstlock)) = do
+		mfo <- fa file firstlock
 		case mfo of
 			Nothing -> do
-				liftIO $ cleanup (Just ph)
+				liftIO $ cleanup (Just (ph, firstlock))
 				return Nothing
 			Just fo -> liftIO $ Just <$> mkLockHandle ph fo
 
@@ -86,4 +86,4 @@ mkLockHandle :: P.LockHandle -> FileLockOps -> IO LockHandle
 mkLockHandle ph fo = do
 	atomically $ P.registerCloseLockFile ph (fDropLock fo)
 	return $ LockHandle ph fo
-	
+
diff --git a/Utility/LockPool/PidLock.hs b/Utility/LockPool/PidLock.hs
index 02f079a030..d16509074e 100644
--- a/Utility/LockPool/PidLock.hs
+++ b/Utility/LockPool/PidLock.hs
@@ -1,6 +1,6 @@
 {- Pid locks, using lock pools.
  -
- - Copyright 2015-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2021 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -26,7 +26,9 @@ import Utility.ThreadScheduler
 
 import System.IO
 import System.Posix
+import Control.Concurrent.STM
 import Data.Maybe
+import Control.Monad
 import Control.Monad.Catch
 import Control.Monad.IO.Class
 import Control.Applicative
@@ -43,13 +45,35 @@ waitLock timeout file displaymessage = makeLockHandle P.lockPool file
 	-- LockShared for STM lock, because a pid lock can be the top-level
 	-- lock with various other STM level locks gated behind it.
 	(\p f -> P.waitTakeLock p f LockShared)
-	(\f -> mk <$> F.waitLock timeout f displaymessage)
+	(\f (P.FirstLock firstlock firstlocksem) -> mk 
+		<$> if firstlock
+			then F.waitLock timeout f displaymessage $
+				void . atomically . tryPutTMVar firstlocksem . P.FirstLockSemWaited
+			else liftIO (atomically $ readTMVar firstlocksem) >>= \case
+				P.FirstLockSemWaited True -> F.alreadyLocked f
+				P.FirstLockSemTried True -> F.alreadyLocked f
+				P.FirstLockSemWaited False -> F.waitedLock timeout f displaymessage
+				P.FirstLockSemTried False -> F.waitLock timeout f displaymessage $
+					void . atomically . tryPutTMVar firstlocksem . P.FirstLockSemWaited
+	)
 
 -- Tries to take a pid lock, but does not block.
 tryLock :: LockFile -> IO (Maybe LockHandle)
 tryLock file = tryMakeLockHandle P.lockPool file
 	(\p f -> P.tryTakeLock p f LockShared)
-	(\f -> fmap mk <$> F.tryLock f)
+	(\f (P.FirstLock firstlock firstlocksem) -> fmap mk
+		<$> if firstlock
+			then do
+				lh <- F.tryLock f
+				void $ atomically $ tryPutTMVar firstlocksem 
+					(P.FirstLockSemTried (isJust lh))
+				return lh
+			else liftIO (atomically $ readTMVar firstlocksem) >>= \case

(Diff truncated)

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
index 9854151af8..f809296447 100644
--- a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
@@ -56,3 +56,5 @@ per my setting in `~/.gitconfig` (which also has `annex.retry = 3`)
 [[!meta author=yoh]]
 [[!tag projects/datalad]]
 
+
+[[!meta title="annex.pidlock can cause git-annex get -J to fail to take transfer lock"]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment
index 3550d2b5d6..6d5e6883b4 100644
--- a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment
@@ -53,4 +53,9 @@ sometimes unncessarily failing. Both need to be fixed.
 I am leaning toward a process only taking the pid lock once and holding it
 until exit, with LockPool used to make that thread safe. And add fine grained
 locking using LockPool when doing pid locking.
+
+Utility.LockPool.PidLock does use a LockPool for the pid lock
+already, but with LockShared, which seems rather pointless since it will never make
+any thread block and multiple threads can still try to make the pid lock
+file at the same time.
 """]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment
new file mode 100644
index 0000000000..3550d2b5d6
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_2_5da78a05b781029fa7f0f9d8ead7e093._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-12-01T17:05:14Z"
+ content="""
+What's happening is Utility.LockFile.PidLock.tryLock
+calls trySideLock and in these failure cases,
+that returns Nothing, because another thread also happens to
+have taken the sidelock. 
+
+Due to the concurrency, the pid lock file always already exists, so
+linkToLock fails. When it is able to take the side lock, it treats this as
+a stale pid lock file situation, and takes over the pid lock.
+
+(This could also affect other locks than transfer locks, potentially.)
+
+One way to solve it would be to use a LockPool instead to take the side
+lock. Then multiple concurrent threads could all lock the side lock.
+
+Or, it could special case when the pid in the pid lock file is the same as
+the current pid, and handle it in the lock file take over code.
+
+----
+
+Now, annex.pidlock is supposed to be a big top-level lock, which is used
+instead of the fine-grained locking. What if 2 threads are each wanting
+to take a lock before operating on the same resource? If either of the
+solutions above is implemented, then both threads will "succeed" at locking
+even though it's a shared pidlock. Which could result in any undefined
+behavior.
+
+And, in all the cases where it does *not* fail to take the transfer lock,
+but instead takes over the pid lock, we're perilously close to such a thing
+happening! The only reason it's not a problem, in the case of transfers
+is that OnlyActionOn is used and prevents two threads transferring the same
+key. (Also used for dropping etc.)
+
+But this makes me worry that using annex.pidlock with concurrency enabled
+is flirting with disaster, and perhaps it should refuse to use concurrency
+to avoid such potential problems. Unless there's a way to avoid them
+entirely.
+
+Hmm.. Normally all locking in git-annex uses LockPool, which
+handles inter-process locking. If LockPool is used, one of those 2 threads
+will win at the lock and the other one will wait for it. But,
+Annex.LockPool.PosixOrPid.tryPidLock/pidLock do not use LockPool
+for fine-grained locking when pid locking is enabled.
+
+So, I think it's potentially buggy in the unsafe direction of letting
+2 threads "lock" the same resource, as well as in the safe direction of
+sometimes unncessarily failing. Both need to be fixed.
+
+I am leaning toward a process only taking the pid lock once and holding it
+until exit, with LockPool used to make that thread safe. And add fine grained
+locking using LockPool when doing pid locking.
+"""]]

diff --git a/doc/todo/incremental_hashing_for_add/comment_2_30b4c0f3b1c13e9a1a2ddba5f88eb6ee._comment b/doc/todo/incremental_hashing_for_add/comment_2_30b4c0f3b1c13e9a1a2ddba5f88eb6ee._comment
new file mode 100644
index 0000000000..caaffbfc6b
--- /dev/null
+++ b/doc/todo/incremental_hashing_for_add/comment_2_30b4c0f3b1c13e9a1a2ddba5f88eb6ee._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-12-01T16:59:50Z"
+ content="""
+Thanks @adina.wagner that is very useful. Seems reliable enough despite the
+other workload, to say that the overhead on Windows is around 14%-33%.
+Interesting that the worst case is with the smallest file.
+"""]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_1_4de90238d57de0d49b75418411135dbc._comment b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_1_4de90238d57de0d49b75418411135dbc._comment
new file mode 100644
index 0000000000..4262fafb50
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3/comment_1_4de90238d57de0d49b75418411135dbc._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-12-01T16:39:05Z"
+ content="""
+It seems that every failure in that log is preceeded by "transfer already
+in progress, or unable to take transfer lock".
+
+It does not make sense to retry in such a case because the transfer will
+probably still be in progress so the retry would fail.
+
+So I don't think this really involves retrying (which certianly supports
+S3). The question is why are several transfers failing this way.
+My suspicion would be something to do with pid locking.
+
+And indeed, I can reproduce the same behavior in a local repo getting from
+another local repo, once I set annex.pidlock.
+"""]]

diff --git a/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
new file mode 100644
index 0000000000..9854151af8
--- /dev/null
+++ b/doc/bugs/annex_get_should_retry_failed_downloads_from_S3.mdwn
@@ -0,0 +1,58 @@
+### Please describe the problem.
+
+IMHO `git -c annex.retry=5 annex get -J...` should do its really best (up to 5 times per file) to obtain content.
+It is really not pragmatic to demand looping on top of such a call to ensure that eventually we do get all content.
+
+Possibly relevant older issues/todos I ran into:
+- [https://git-annex.branchable.com/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/](https://git-annex.branchable.com/forum/is_there_a_way_to_automatically_retry_when_special_remotes_fail__63__/)
+- [https://git-annex.branchable.com/todo/more_extensive_retries_to_mask_transient_failures/](https://git-annex.branchable.com/todo/more_extensive_retries_to_mask_transient_failures/)
+
+
+### What steps will reproduce the problem?
+
+I have cloned `http://datasets.datalad.org/labs/hasson/narratives/derivatives/freesurfer/.git` and did `git annex get -J5 . 2>&1 | tee /tmp/annex-get-freesurfer-patched-annex-1.log` whenever 
+
+```
+[d31548v@discovery7 freesurfer]$ git config annex.retry
+3
+```
+
+that log file is available from http://www.onerussian.com/tmp/annex-get-freesurfer-patched-annex-1.log . That call ended with `get: 2948 failed`.
+
+I have dumped the output of `annex list` into [a file](http://www.onerussian.com/tmp/annex-get-freesurfer-patched-annex-1-listafter.out) and now ran
+
+```
+grep '^_' /tmp/annex-get-freesurfer-patched-annex-1-listafter.out | awk '{print $2;}' | xargs git -c annex.retry=10 annex --debug get -J5 2>&1 | tee /tmp/annex-get-freesurfer-patched-annex-1-getabsent.log
+```
+
+log for which is at [http://www.onerussian.com/tmp/annex-get-freesurfer-patched-annex-1-getabsent.log](http://www.onerussian.com/tmp/annex-get-freesurfer-patched-annex-1-getabsent.log)
+
+and which finished with `get: 130 failed` -- so clearly those original failures were not real inability to fetch content.
+
+### What version of git-annex are you using? On what operating system?
+
+
+```
+[d31548v@discovery7 git-annexes]$ git annex version
+git-annex version: 8.20211123+git21-ga6699be79-1~ndall+1
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+```
+
+and this is on that "fancy" NFS mounted isilon on a local HPC (a "POSIX" mount this time).
+
+```
+[d31548v@discovery7 freesurfer]$ git config annex.pidlock
+true
+```
+
+per my setting in `~/.gitconfig` (which also has `annex.retry = 3`)
+
+[[!meta author=yoh]]
+[[!tag projects/datalad]]
+

diff --git a/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_1_aca3cd6a0d8ed7e802140590a7d0972d._comment b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_1_aca3cd6a0d8ed7e802140590a7d0972d._comment
new file mode 100644
index 0000000000..b2eb56843d
--- /dev/null
+++ b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1/comment_1_aca3cd6a0d8ed7e802140590a7d0972d._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="account@dc612ad075297e574ebc3eb9a5b8ab6e753510dc"
+ nickname="account"
+ avatar="http://cdn.libravatar.org/avatar/6a2e457685ce943a3d85b0a6d1ffb11a"
+ subject="Further fix attempts"
+ date="2021-12-01T03:25:35Z"
+ content="""
+I made another attempt at a fix by porting whatever changes you made to stack.yaml on 2739adc to git-annex.cabal using the following patch.
+[[!format sh \"\"\"
+From 05030463daca2582f2512cf14515393971263a40 Mon Sep 17 00:00:00 2001
+From: \"build@apk-groulx\" <build@apk-groulx.praxis>
+Date: Tue, 30 Nov 2021 06:48:09 +0000
+Subject: [PATCH 1/1] Added dependency for ghc 9.0.1 for cabal
+
+---
+ git-annex.cabal | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/git-annex.cabal b/git-annex.cabal
+index 9285858cd..1a3ee52ff 100644
+--- a/git-annex.cabal
++++ b/git-annex.cabal
+@@ -330,7 +330,9 @@ Executable git-annex
+    directory (>= 1.2),
+    disk-free-space,
+    filepath,
+-   filepath-bytestring (>= 1.4.2.1.1),
++   filepath-bytestring (>= 1.4.2.1.8),
++   base16-bytestring (>=0.1.1.7),
++   base64-bytestring (>=0.0.3),
+    IfElse,
+    monad-logger (>= 0.3.10),
+    free,
+@@ -425,7 +427,7 @@ Executable git-annex
+     Other-Modules: Utility.GitLFS
+   
+   if flag(HttpClientRestricted)
+-    Build-Depends: http-client-restricted (>= 0.0.2)
++    Build-Depends: http-client-restricted (>= 0.0.4)
+     CPP-Options: -DWITH_HTTP_CLIENT_RESTRICTED
+   else
+     Other-Modules: Utility.HttpManagerRestricted
+-- 
+2.34.0
+\"\"\"]]
+
+The same error occurs, unfortunately.
+"""]]

diff --git a/doc/todo/incremental_hashing_for_add/comment_1_d00827f5edd7b270aa6dfc011e7c9f22._comment b/doc/todo/incremental_hashing_for_add/comment_1_d00827f5edd7b270aa6dfc011e7c9f22._comment
new file mode 100644
index 0000000000..690fb7d1e7
--- /dev/null
+++ b/doc/todo/incremental_hashing_for_add/comment_1_d00827f5edd7b270aa6dfc011e7c9f22._comment
@@ -0,0 +1,449 @@
+[[!comment format=mdwn
+ username="adina.wagner@2a4cac6443aada2bd2a329b8a33f4a7b87cc8eff"
+ nickname="adina.wagner"
+ avatar="http://cdn.libravatar.org/avatar/80b124ad61d6008fa0f6f0b4b0f7c2ef"
+ subject="A few Windows benchmarks"
+ date="2021-11-29T22:17:39Z"
+ content="""
+Following 
+ [https://git-annex.branchable.com/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/#comment-7e6fd13d8b60d99cbc5e642e1d76f10e](https://git-annex.branchable.com/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/#comment-7e6fd13d8b60d99cbc5e642e1d76f10e) I've run a few quick benchmarks for ``git add`` with and without the filter.annex.process setting.
+I'm using a Levono Thinkpad T14 with Windows 10 Pro Version 20H2, OS build 19042.1348. It has a AMD Ryzen 7 PRO 4750U with Radeon Graphics 1.70 GHz and 16GB RAM. It ran on battery when I tested this.
+
+<details><summary> Relevant software </summary>
+
+git-annex:
+
+```
+git-annex version: 8.20211117-gc3af94eff
+build flags: Assistant Webapp Pairing TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: mingw32 x86_64
+supported repository versions: 8
+upgrade supported from repository versions: 2 3 4 5 6 7
+```
+
+Git:
+
+```
+git version 2.34.0.windows.1
+cpu: x86_64
+built from commit: 7dad2af5e8ae15d5f1e1ce1980d1d873b9e00cb5
+sizeof-long: 4
+sizeof-size_t: 8
+shell-path: /bin/sh
+feature: fsmonitor--daemon
+```
+
+The machine is pretty much brand new, there is no Antivirus, and very little other software other than git and git-annex. [It is sometimes moody](https://github.com/datalad/datalad/issues/5994#issuecomment-974205491) but today felt fast.
+
+</details>
+
+I created files of different sizes from 1MB to 50GB; added them sequentially into the same repo; on a busy machine running the datalad test suite in the background. There are more timings for \"with config\" because there was an additional global setting I forgot to remove initially, and I only added really large files in a later runs, so I have a few more timings for smaller files.
+
+<details><summary> What I ran </summary>
+
+```
+(sorry for this embarrassing script - I hate windows and have no clue how to use it productively)
+git config --system --unset  filter.annex.process
+git config --list
+git init withoutconfig
+cd withoutconfig
+git annex init 
+fsutil file createnew 1MB 1048576
+fsutil file createnew 100MB 104857600 
+fsutil file createnew 500MB 524288000
+fsutil file createnew 1GB 1073741824
+fsutil file createnew 5GB 5368709120
+fsutil file createnew 10GB 10737418240
+fsutil file createnew 20GB 21474836480
+fsutil file createnew 50GB 53687091200
+ptime git -c annex.largefiles=anything add 1MB
+ptime git -c annex.largefiles=anything add 100MB
+ptime git -c annex.largefiles=anything add 500MB
+ptime git -c annex.largefiles=anything add 1GB
+ptime git -c annex.largefiles=anything add 5GB
+ptime git -c annex.largefiles=anything add 10GB
+ptime git -c annex.largefiles=anything add 20GB
+ptime git -c annex.largefiles=anything add 50GB
+cd ..
+
+rmdir /s withoutconfig
+
+git config --system filter.annex.process \"git-annex filter-process\"
+git config --list
+git init withconfig
+cd withconfig
+git annex init 
+fsutil file createnew 1MB 1048576
+fsutil file createnew 100MB 104857600 
+fsutil file createnew 500MB 524288000
+fsutil file createnew 1GB 1073741824
+fsutil file createnew 5GB 5368709120
+fsutil file createnew 10GB 10737418240
+fsutil file createnew 20GB 21474836480
+fsutil file createnew 50GB 53687091200
+ptime git -c annex.largefiles=anything add 1MB
+ptime git -c annex.largefiles=anything add 100MB
+ptime git -c annex.largefiles=anything add 500MB
+ptime git -c annex.largefiles=anything add 1GB
+ptime git -c annex.largefiles=anything add 5GB
+ptime git -c annex.largefiles=anything add 10GB
+ptime git -c annex.largefiles=anything add 20GB
+ptime git -c annex.largefiles=anything add 50GB
+
+rmdir /s withconfig
+```
+</details>
+
+**without config:**
+
+- 1MB -> 0.969s / 1.027s
+- 100MB -> 1.676s / 1.760s
+- 500MB -> 4.607s / 4.645s
+- 1GB -> 5.473s / 5.834s
+- 5GB -> 29.869s / 26.916s
+- 10GB -> 54.708s / 54.275
+- 20GB -> 104.860s / 105.074s
+- 50GB -> 313.757s / 256.739 s
+
+**with config:**
+
+- 1MB ->  0.316s / 0.302s / 0.068s / 1.012s
+- 100MB -> 1.006s / 0.951s / 1.033s / 1.875s
+- 500MB -> 3.141s / 3.417s / 4.762s / 5.131s
+- 1GB -> 7.257s / 7.11s / 9.610s / 6.053s
+- 5GB ->  34.450s / 36.78 / 4.852s (really no clue why, not a typo) / 34.925s
+- 10GB -> 62.202 / 76.007s
+- 20GB -> 127.382s / 135.851s
+- 50GB -> 374.773 / 350.991s
+
+
+<details><summary> example run protocol </summary>
+
+
+```
+C:\Users\adina\temp\gitannexbenchmarks>git config --system filter.annex.process \"git-annex filter-process\"
+
+C:\Users\adina\temp\gitannexbenchmarks>git config --list
+diff.astextplain.textconv=astextplain
+filter.lfs.clean=git-lfs clean -- %f
+filter.lfs.smudge=git-lfs smudge -- %f
+filter.lfs.process=git-lfs filter-process
+filter.lfs.required=true
+http.sslbackend=openssl
+http.sslcainfo=C:/Program Files/Git/mingw64/ssl/certs/ca-bundle.crt
+core.autocrlf=true
+core.fscache=true
+core.symlinks=true
+pull.rebase=false
+credential.helper=manager-core
+credential.https://dev.azure.com.usehttppath=true
+init.defaultbranch=master
+filter.annex.process=git-annex filter-process
+user.email=adina.wagner@t-online.de
+user.name=Adina Wagner
+hub.oauthtoken=cd2a3bdefd55af090f495e9f36454f73b5a404dc
+alias.co=checkout
+alias.st=status
+core.symlinks=true
+core.fscache=true
+filter.annex.process=git-annex filter-process
+
+C:\Users\adina\temp\gitannexbenchmarks>
+C:\Users\adina\temp\gitannexbenchmarks>git init withconfig
+Initialized empty Git repository in C:/Users/adina/temp/gitannexbenchmarks/withconfig/.git/
+
+C:\Users\adina\temp\gitannexbenchmarks>cd withconfig
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>git annex init
+init
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Disabling core.symlinks.
+
+  Entering an adjusted branch where files are unlocked as this filesystem does not support locked files.
+
+Switched to branch 'adjusted/master(unlocked)'
+ok
+(recording state in git...)
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 1MB 1048576
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\1MB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 100MB 104857600
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\100MB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 500MB 524288000
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\500MB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 1GB 1073741824
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\1GB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 5GB 5368709120
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\5GB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 10GB 10737418240
+File C:\Users\adina\temp\gitannexbenchmarks\withconfig\10GB is created
+
+C:\Users\adina\temp\gitannexbenchmarks\withconfig>fsutil file createnew 20GB 21474836480

(Diff truncated)

diff --git a/Utility/LockFile/PidLock.hs b/Utility/LockFile/PidLock.hs
index c4e80f7749..4929c42d56 100644
--- a/Utility/LockFile/PidLock.hs
+++ b/Utility/LockFile/PidLock.hs
@@ -154,14 +154,11 @@ tryLock lockfile = do
 			removeWhenExistsWith removeLink tmp'
 			return Nothing
 		let tooklock st = return $ Just $ LockHandle abslockfile st sidelock
-		ifM (linkToLock sidelock tmp' abslockfile)
-			( do
+		linkToLock sidelock tmp' abslockfile >>= \case
+			Just lckst -> do
 				removeWhenExistsWith removeLink tmp'
-				-- May not have made a hard link, so stat
-				-- the lockfile
-				lckst <- getFileStatus abslockfile
 				tooklock lckst
-			, do
+			Nothing -> do
 				v <- readPidLock abslockfile
 				hn <- getHostName
 				tmpst <- getFileStatus tmp'
@@ -175,7 +172,6 @@ tryLock lockfile = do
 						rename tmp' abslockfile
 						tooklock tmpst
 					_ -> failedlock tmpst
-			)
 
 -- Linux's open(2) man page recommends linking a pid lock into place,
 -- as the most portable atomic operation that will fail if
@@ -187,8 +183,8 @@ tryLock lockfile = do
 --
 -- However, not all filesystems support hard links. So, first probe
 -- to see if they are supported. If not, use open with O_EXCL.
-linkToLock :: SideLockHandle -> RawFilePath -> RawFilePath -> IO Bool
-linkToLock Nothing _ _ = return False
+linkToLock :: SideLockHandle -> RawFilePath -> RawFilePath -> IO (Maybe FileStatus)
+linkToLock Nothing _ _ = return Nothing
 linkToLock (Just _) src dest = do
 	let probe = src <> ".lnk"
 	v <- tryIO $ createLink src probe
@@ -197,10 +193,13 @@ linkToLock (Just _) src dest = do
 		Right _ -> do
 			_ <- tryIO $ createLink src dest
 			ifM (catchBoolIO checklinked)
-				( catchBoolIO $ not <$> checkInsaneLustre dest
-				, return False
+				( ifM (catchBoolIO $ not <$> checkInsaneLustre dest)
+					( catchMaybeIO $ getFileStatus dest
+					, return Nothing
+					)
+				, return Nothing
 				)
-		Left _ -> catchBoolIO $ do
+		Left _ -> catchMaybeIO $ do
 			let setup = do
 				fd <- openFd dest WriteOnly
 					(Just $ combineModes readModes)
@@ -209,7 +208,7 @@ linkToLock (Just _) src dest = do
 			let cleanup = hClose
 			let go h = readFile (fromRawFilePath src) >>= hPutStr h
 			bracket setup cleanup go
-			return True
+			getFileStatus dest
   where
 	checklinked = do
 		x <- getSymbolicLinkStatus src
diff --git a/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist/comment_1_83448258af56b5872a9530cf60c72eb0._comment b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist/comment_1_83448258af56b5872a9530cf60c72eb0._comment
new file mode 100644
index 0000000000..d86821d32b
--- /dev/null
+++ b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist/comment_1_83448258af56b5872a9530cf60c72eb0._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-29T18:11:30Z"
+ content="""
+I think that this error comes from Utility.LockFile.PidLock.tryLock,
+which has the only getFileStatus involving the pidlock whose exceptions
+are not caught. The file is assumed to exist since it was just created,
+and normally nothing deletes it.
+
+While looking at where this might come from, I refreshed my memory of
+how Lustre can to do insane stuff like having 2 different files with the
+same name in a directory. Which checkInsaneLustre tries to deal with
+by deleting one of them, but since this is all behavior undefined by POSIX,
+maybe that sometimes deletes both of them. Or the file doesn't appear
+after being created for some other POSIX-defying reason.
+
+I've changed it to catch exceptions from that getFileStatus, which will
+test this theory.
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index ae0c384bb2..795bb3de4d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -4,6 +4,8 @@ git-annex (8.20211124) UNRELEASED; urgency=medium
   * addurl, youtube-dl: When --check-raw prevents downloading an url,
     still continue with any downloads that come after it, rather than
     erroring out.
+  * export: Avoid unncessarily re-exporting non-annexed files that were
+    already exported.
 
  -- Joey Hess <id@joeyh.name>  Tue, 23 Nov 2021 15:58:27 -0400
 
diff --git a/Command/Export.hs b/Command/Export.hs
index b94f12c3d2..bb848f42f1 100644
--- a/Command/Export.hs
+++ b/Command/Export.hs
@@ -275,11 +275,20 @@ startExport r db cvar allfilledvar ti = do
 	af = AssociatedFile (Just f)
 	ai = ActionItemTreeFile f
 	si = SeekInput []
-	notrecordedpresent ek = (||)
-		<$> liftIO (notElem loc <$> getExportedLocation db ek)
-		-- If content was removed from the remote, the export db
-		-- will still list it, so also check location tracking.
-		<*> (notElem (uuid r) <$> loggedLocations ek)
+	notrecordedpresent ek = 
+		ifM  (liftIO $ notElem loc <$> getExportedLocation db ek)
+			( return True
+			-- When content was lost from the remote and
+			-- a fsck noticed that, the export db will still
+			-- list it as present in the remote. So also check 
+			-- location tracking. 
+			-- However, git sha keys do not have their locations
+			-- tracked, and fsck doesn't check them, so not
+			-- for those.
+			, if isGitShaKey ek
+				then return False
+				else notElem (uuid r) <$> loggedLocations ek
+			)
 
 performExport :: Remote -> ExportHandle -> Key -> AssociatedFile -> Sha -> ExportLocation -> MVar AllFilled -> CommandPerform
 performExport r db ek af contentsha loc allfilledvar = do
diff --git a/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn b/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn
index b2527070fe..6e99c355a5 100644
--- a/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn
+++ b/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn
@@ -27,3 +27,4 @@ git annex from Debian :
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 It’s only a bug on what git-annex display. It does the asked work: the file are correctly exported where I look for them.
 
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git_annex_try_to_rexeport_already_exported_files/comment_1_93fd0ded251371e34cd113f27e75ecc2._comment b/doc/bugs/git_annex_try_to_rexeport_already_exported_files/comment_1_93fd0ded251371e34cd113f27e75ecc2._comment
new file mode 100644
index 0000000000..d84152d121
--- /dev/null
+++ b/doc/bugs/git_annex_try_to_rexeport_already_exported_files/comment_1_93fd0ded251371e34cd113f27e75ecc2._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-29T17:32:33Z"
+ content="""
+I think that the most likely reason for this to happen is if the 
+particular file is not checked into git-annex, but into git.
+
+Since files stored in git are small, re-uploading them is not likely to be
+very expensive.
+
+However, I found a way to prevent it doing so. So this is fixed assuming
+I guessed right about the cause of it for you.
+"""]]

diff --git a/doc/todo/incremental_hashing_for_add.mdwn b/doc/todo/incremental_hashing_for_add.mdwn
index bb37c65511..3708780964 100644
--- a/doc/todo/incremental_hashing_for_add.mdwn
+++ b/doc/todo/incremental_hashing_for_add.mdwn
@@ -17,7 +17,7 @@ and moving it into place there, because annex.thin means a hard link should
 be made, and if the filesystem supports CoW, that should be used rather
 than writing the file again.
 
-A benchmark showed that `git add` of a 1 gb file 
+A benchmark on Linux showed that `git add` of a 1 gb file 
 is about 5% slower with filter-process enabled than it is 
 with filter-process disabled. That's due to the piping overhead to 
 filter-process ([[todo/git_smudge_clean_interface_suboptiomal]]).

diff --git a/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_21_cc08c222cc36e3a0462f370df12eb129._comment b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_21_cc08c222cc36e3a0462f370df12eb129._comment
new file mode 100644
index 0000000000..3c08a42faa
--- /dev/null
+++ b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_21_cc08c222cc36e3a0462f370df12eb129._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 21"""
+ date="2021-11-29T17:17:29Z"
+ content="""
+@mih I've been pondering enabling filter.annex.process by default in new
+repositories in Windows or generally. (Enabling it in existing repositories
+kind of needs a new major version so an upgrade can set it, although introducing
+repository minor versions is also a possibility.)
+
+Enabling it earlier in datalad is fine by me, more experience with it being
+used would be good.
+
+It would also be useful to get some benchmarks of `git add` when 
+large files are added to the annex (eg `git -c annex.largefiles=anything
+add`). As I said, that suffers around 5% on performance on Linux, at least
+when the files are small enough to still mostly fit in disk cache (1 gb on
+a 4 gb system with some web browsers etc running). It may be
+that Windows will pay a higher price. I don't have real Windows machines
+to run such a benchmark on myself. Please post any such benchmarks
+to [[todo/incremental_hashing_for_add]].
+"""]]

diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_21_3f2ce7b851392ad5285e95f3857d0959._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_21_3f2ce7b851392ad5285e95f3857d0959._comment
new file mode 100644
index 0000000000..e4d12df11f
--- /dev/null
+++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_21_3f2ce7b851392ad5285e95f3857d0959._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 21"""
+ date="2021-11-29T17:09:33Z"
+ content="""
+This seems to be a not entirely uncommon git failure.
+[google search](https://www.google.com/search?client=firefox-b-1-d&q=git+error%3A+invalid+object+100644+)
+has tens of thousands of hits that look like this and do not involve
+git-annex.
+
+One commenter in <https://stackoverflow.com/questions/14448326/git-commit-stopped-working-error-building-trees>
+said it was a permissions problem.
+
+Disconnecting a drive before all of git's changes have been written to it
+seem like the other likely way.
+"""]]

diff --git a/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_4_e854aae525b7d4902fe2bc0f2c11ae5a._comment b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_4_e854aae525b7d4902fe2bc0f2c11ae5a._comment
new file mode 100644
index 0000000000..d2a18ea048
--- /dev/null
+++ b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_4_e854aae525b7d4902fe2bc0f2c11ae5a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2021-11-29T16:57:32Z"
+ content="""
+The successful test log you linked to does have the "migrate" 
+tests in it. Those tests are not new.
+
+Is the test suite being run under a timeout?
+"""]]

diff --git a/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_20_281ded4caa0dd407431b205ce6746c05._comment b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_20_281ded4caa0dd407431b205ce6746c05._comment
new file mode 100644
index 0000000000..39230325a6
--- /dev/null
+++ b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_20_281ded4caa0dd407431b205ce6746c05._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="mih"
+ avatar="http://cdn.libravatar.org/avatar/f881df265a423e4f24eff27c623148fd"
+ subject="Even more impact on real systems"
+ date="2021-11-26T14:23:34Z"
+ content="""
+It seems the impact on real systems is even more sever: https://github.com/datalad/datalad/pull/6245#issuecomment-979992551 -- factor 2-3 faster on that particular machine. Maybe consumer systems do some kind of expensive process startup inspection (antivirus?) that bring a swarm of child processes to a crawl.
+"""]]

diff --git a/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_20_f12f7f14ef09fa765c78dbeef809da7d._comment b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_20_f12f7f14ef09fa765c78dbeef809da7d._comment
new file mode 100644
index 0000000000..2a44d60f9b
--- /dev/null
+++ b/doc/bugs/__34__error__58___invalid_object__34____44___after_add__59___cannot_commit/comment_20_f12f7f14ef09fa765c78dbeef809da7d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/d1f0f4275931c552403f4c6707bead7a"
+ subject="comment 20"
+ date="2021-11-25T19:23:46Z"
+ content="""
+Also ran across this today. All I did was a simple copy of a few files to a remote using a filter. I can provide the index and lck files if needed.
+
+    git-annex version: 8.20211011
+"""]]

diff --git a/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_19_c89853eca5c16e9dfa4a4acc982ab526._comment b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_19_c89853eca5c16e9dfa4a4acc982ab526._comment
new file mode 100644
index 0000000000..ffbf19ebf9
--- /dev/null
+++ b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_19_c89853eca5c16e9dfa4a4acc982ab526._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="mih"
+ avatar="http://cdn.libravatar.org/avatar/f881df265a423e4f24eff27c623148fd"
+ subject="More statistics"
+ date="2021-11-25T13:09:11Z"
+ content="""
+I looked into the global affect of this switch on a large and versatile set of use cases in the form of the DataLad test suite: https://github.com/datalad/datalad/pull/6245
+
+It is worth keeping in mind that there are only small-size files involved!
+
+The benefit is somewhere between noticeable and remarkable. An overall runtime reduction of 16% with benefits ranging from 5% to 32% depending on the tested functionality.
+
+This will need a bit of further investigation to drill down on the reason for this large variability, but given that the sign is always in the right direction this is really great! Thx much!
+"""]]

diff --git a/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn b/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn
new file mode 100644
index 0000000000..b2527070fe
--- /dev/null
+++ b/doc/bugs/git_annex_try_to_rexeport_already_exported_files.mdwn
@@ -0,0 +1,29 @@
+### Please describe the problem.
+I've a type=directory remote, with exporttree=yes on a USB disk. when running `git annex --content` git annex list a long list of `export cibox-usb toto ok` lines for file it already exported before. I’ve check some of those file, and their content are correct.
+
+Those line hide real new export, and I wonder if it doesn’t slow the process.
+
+There seem to be no way to tell git-annex that those file are already ok, and it should try again to export them.
+`
+
+### What steps will reproduce the problem?
+
+Just running `git annex --content name-of-the-remote`
+
+
+### What version of git-annex are you using? On what operating system?
+git annex from Debian :
+
+    git-annex version: 8.20211011
+    build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+    dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512     BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+    operating system: linux x86_64
+    supported repository versions: 8
+    upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+    local repository version: 
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+It’s only a bug on what git-annex display. It does the asked work: the file are correctly exported where I look for them.
+

diff --git a/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_18_2e8e840cabbc547ef758568bb2779f9a._comment b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_18_2e8e840cabbc547ef758568bb2779f9a._comment
new file mode 100644
index 0000000000..22fbac94a1
--- /dev/null
+++ b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_18_2e8e840cabbc547ef758568bb2779f9a._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="mih"
+ avatar="http://cdn.libravatar.org/avatar/f881df265a423e4f24eff27c623148fd"
+ subject="Translates to Windows!"
+ date="2021-11-25T07:34:48Z"
+ content="""
+Sorry for being silent for so long. I just got a contemporary machine with windows, such that hardware age should no longer be a concern for any performance comparison.
+
+I did not yet find the time to re-assess this issue in full, but I tried the new filter-process setting with a simple `datalad create` (this only adds a few tiny files, but nevertheless took long, and was the original motivation for this issue). Enabling the new setting reduces the runtime by 25% (from 4.5s to 3.5s on average).
+"""]]

diff --git a/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn
new file mode 100644
index 0000000000..758defb8cf
--- /dev/null
+++ b/doc/bugs/Remote__47__Glacier.hs_build_error_GHC_9.0.1.mdwn
@@ -0,0 +1,52 @@
+### Please describe the problem.
+Build error 1 during build process of git-annex on alpine 5.15 using GHC 9.0.1.
+
+### What steps will reproduce the problem?
+Build git-annex on alpine 5.15 using following APKBUILD:
+https://github.com/ayakael/aports/blob/testing/git-annex/testing/git-annex/APKBUILD
+
+### What version of git-annex are you using? On what operating system?
+git-annex: 20211123
+os: alpine linux 5.15
+ghc: 9.0.1
+cabal: 3.6.2.0
+
+### Please provide any additional information below.
+I made an attempt of fixing by reverting the first fix attempt done on line 181 by commit 2739adc. It gave the same error, but it is most likely related. I know very little of haskell, thus could not push the debugging further.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+[411 of 670] Compiling Remote.Glacier   ( Remote/Glacier.hs, /var/build/aports/testing/git-annex/src/git-annex-8.20211123/dist-newstyle/build/x86_64-linux/ghc-9.0.1/git-annex-8.20211123/build/git-annex/git-annex-tmp/Remote/Glacier.o, /var/build/aports/testing/git-annex/src/git-annex-8.20211123/dist-newstyle/build/x86_64-linux/ghc-9.0.1/git-annex-8.20211123/build/git-annex/git-annex-tmp/Remote/Glacier.dyn_o )
+
+Remote/Glacier.hs:179:12: error:
+    • Couldn't match type: Key
+                           -> MeterUpdate
+                           -> Maybe Utility.Hash.IncrementalVerifier
+                           -> (ContentSource -> Annex a0)
+                           -> Annex a0
+                     with: forall a.
+                           Key
+                           -> MeterUpdate
+                           -> Maybe Utility.Hash.IncrementalVerifier
+                           -> (ContentSource -> Annex a)
+                           -> Annex a
+      Expected: Remote -> Retriever
+        Actual: Remote
+                -> Key
+                -> MeterUpdate
+                -> Maybe Utility.Hash.IncrementalVerifier
+                -> (ContentSource -> Annex a0)
+                -> Annex a0
+    • In the expression: byteRetriever . retrieve'
+      In an equation for ‘retrieve’: retrieve = byteRetriever . retrieve'
+    |
+179 | retrieve = byteRetriever . retrieve'
+    |            ^^^^^^^^^^^^^^^^^^^^^^^^^
+make: *** [Makefile:58: git-annex] Error 1
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Oh absolutely. This version compiles fine under alpine 5.14, which uses GHC 8.8.4. I've been using this software for many years without major issues, and it's a linchpin of my backup infrastructure. Thanks for the all the good work!
+

diff --git a/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn
index 36b5dbbab6..917748b90b 100644
--- a/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn
+++ b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn
@@ -15,3 +15,5 @@ conda linux nodep (standalone) build 8.20211012-geb95ed486 and then with another
 
 
 
+[[!meta author=yoh]]
+[[!tag projects/datalad]]

diff --git a/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_3_82b8570c9496ee1b111dbd1ea53e823c._comment b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_3_82b8570c9496ee1b111dbd1ea53e823c._comment
new file mode 100644
index 0000000000..b2ad0e8ffb
--- /dev/null
+++ b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_3_82b8570c9496ee1b111dbd1ea53e823c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 3"
+ date="2021-11-23T23:22:36Z"
+ content="""
+oh that [successful log](https://github.com/datalad/git-annex/runs/4271323847?check_suite_focus=true) doesn't even have any line with `migrate`, so some new test stalling on NFS HOME/CWD?
+"""]]

diff --git a/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_2_da46fc7a6bbdae0c3b44d241bc527b55._comment b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_2_da46fc7a6bbdae0c3b44d241bc527b55._comment
new file mode 100644
index 0000000000..0f35fd7057
--- /dev/null
+++ b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_2_da46fc7a6bbdae0c3b44d241bc527b55._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2021-11-23T23:12:47Z"
+ content="""
+I see that. But the entire run was not prohibitively long - just 30m 21s... previous succesful though ran total 22m 14s (see eg [here](https://github.com/datalad/git-annex/runs/4271323847?check_suite_focus=true)) so may be test started to stall?
+"""]]

diff --git a/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_4_50aa0ea762d09990f8a8f633e56aa205._comment b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_4_50aa0ea762d09990f8a8f633e56aa205._comment
new file mode 100644
index 0000000000..9d841b0b28
--- /dev/null
+++ b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_4_50aa0ea762d09990f8a8f633e56aa205._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="comment 4"
+ date="2021-11-23T21:46:43Z"
+ content="""
+Thanks Joey for investigating this issue thoroughly. Also, my thanks to kyle for expanding my initial report
+with Linux-related details I didn't have time to prepare.
+"""]]

diff --git a/doc/news/version_8.20210803.mdwn b/doc/news/version_8.20210803.mdwn
deleted file mode 100644
index 35d9550d3e..0000000000
--- a/doc/news/version_8.20210803.mdwn
+++ /dev/null
@@ -1,27 +0,0 @@
-git-annex 8.20210803 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * whereused: New command, finds what files use a key, or where a key
-    was used historically.
-  * Fix a bug that prevented getting content from a repository that
-    started out as a bare repository, or had annex.crippledfilesystem
-    set, and was converted to a non-bare repository.
-  * Fix retrieval of content from borg repos accessed over ssh.
-  * sync: When --quiet is used, run git commit, push, and pull without
-    their ususual output.
-  * merge: When --quiet is used, run git merge without its usual output.
-  * sync, merge, post-receive: Avoid merging unrelated histories,
-    which used to be allowed only to support direct mode repositories.
-    (However, sync does still merge unrelated histories when importing
-    trees from special remotes, and the assistant still merges unrelated
-    histories.)
-  * sync, merge: Added --allow-unrelated-histories option, which
-    is the same as the git merge option.
-  * Fix bug that caused some transfers to incorrectly fail with
-    "content changed while it was being sent", when the content was not
-    changed.
-  * Fix bug that could prevent pointer files from being populated,
-    in a repository that was upgraded from v7.
-  * fsck: Detect and correct stale or missing inode caches.
-  * Fix a rounding bug in display of data sizes.
-  * git-annex get when run as the first git-annex command in a new repo
-    did not populate unlocked files.
-    (Reversion in version 8.20210621)"""]]
\ No newline at end of file
diff --git a/doc/news/version_8.20211123.mdwn b/doc/news/version_8.20211123.mdwn
new file mode 100644
index 0000000000..f3741d50b4
--- /dev/null
+++ b/doc/news/version_8.20211123.mdwn
@@ -0,0 +1,7 @@
+git-annex 8.20211123 released with [[!toggle text="these changes"]]
+[[!toggleable text="""  * Bugfix: When -J was enabled, getting files could leak an
+    ever-growing number of git cat-file processes.
+  * Support git's new "ort" resolver, which became the default in git 2.34.0,
+    and broke the test suite and automatic merge resolution of a conflict
+    between an annexed file and a non-annexed file.
+  * importfeed: Display url before starting youtube-dl download."""]]
\ No newline at end of file

diff --git a/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_1_dd4734cec6290cd3eacd58545ef039e4._comment b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_1_dd4734cec6290cd3eacd58545ef039e4._comment
new file mode 100644
index 0000000000..9f978ec8a6
--- /dev/null
+++ b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS/comment_1_dd4734cec6290cd3eacd58545ef039e4._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-23T19:14:47Z"
+ content="""
+124 is SIGTERM, and the test process does not complete, the last line
+output is:
+
+	    migrate (via gitattributes):                        
+
+I can reproduce that behavior exactly with `killall -TERM git-annex`.
+
+I think something is timing out on your CI system and it's sending a
+TERM signal.
+"""]]

diff --git a/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn
new file mode 100644
index 0000000000..36b5dbbab6
--- /dev/null
+++ b/doc/bugs/get_-J5___58___pidlock__58___getFileStatus__58___does_not_exist.mdwn
@@ -0,0 +1,17 @@
+### Please describe the problem.
+
+happens while running `git annex get -J5 .` on http://datasets.datalad.org/labs/hasson/narratives/derivatives/fmriprep/.git
+
+filesystem is that NFS mounted lustre where I thought we resolved all the issues (spotted by `annex test`, yet to redo)
+
+the entire run has failed with `get: 128 failed`
+
+It is consistent. Tried in another dataset  http://datasets.datalad.org/labs/hasson/narratives/derivatives/freesurfer/.git/ and after one crashed run (kerberos key timeout, leading to other errors), started a new `annex get -J5` for which log in full is http://www.onerussian.com/tmp/annex-get-freesurfer-5.log and which ends with `get: 89 failed` 
+
+
+### What version of git-annex are you using? On what operating system?
+
+conda linux nodep (standalone) build 8.20211012-geb95ed486 and then with another standalone build (debian, extracted on that CentOS) 8.20211117+git14-ge1f38b9dd  
+
+
+

diff --git a/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS.mdwn b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS.mdwn
new file mode 100644
index 0000000000..44a6dd841c
--- /dev/null
+++ b/doc/bugs/tests___34__exit_code_124__34___with_HOME_and_PWD_on_NFS.mdwn
@@ -0,0 +1,36 @@
+### Please describe the problem.
+
+Tests started to exit with 124 in that testing scenario recently, e.g. [see this log](https://github.com/datalad/git-annex/runs/4294679306?check_suite_focus=true#step:7:29) which also available on smaug under `/mnt/datasets/datalad/ci/git-annex/builds/2021/11/cron-20211123` 
+
+started to fail 3 days ago but passed ones between :
+
+```
+(git)smaug:/mnt/datasets/datalad/ci/git-annex/builds/2021/11[master]cron-20211123
+$> cd ..
+cron-20211101/  cron-20211103/  cron-20211105/  cron-20211107/  cron-20211109/  cron-20211111/  cron-20211113/  cron-20211115/  cron-20211117/  cron-20211119/  cron-20211121/  cron-20211123/
+cron-20211102/  cron-20211104/  cron-20211106/  cron-20211108/  cron-20211110/  cron-20211112/  cron-20211114/  cron-20211116/  cron-20211118/  cron-20211120/  cron-20211122/  manual-20211117/
+
+$> git grep 'exit code 124'
+cron-20211121/build-ubuntu.yaml-490-280a49ed-failed/4_test-annex (nfs-home).txt:2021-11-21T03:23:49.4599000Z ##[error]Process completed with exit code 124.
+cron-20211121/build-ubuntu.yaml-490-280a49ed-failed/test-annex (nfs-home)/7_Run tests.txt:2021-11-21T03:23:49.4598983Z ##[error]Process completed with exit code 124.
+cron-20211123/build-ubuntu.yaml-492-280a49ed-failed/4_test-annex (nfs-home).txt:2021-11-23T03:27:04.5604733Z ##[error]Process completed with exit code 124.
+cron-20211123/build-ubuntu.yaml-492-280a49ed-failed/test-annex (nfs-home)/7_Run tests.txt:2021-11-23T03:27:04.5604714Z ##[error]Process completed with exit code 124.
+
+$> tail cron-20211122/build-ubuntu.yaml-491-280a49ed-failed/test-annex\ \(nfs-home\)/7_Run\ tests.txt
+2021-11-22T03:11:27.7277096Z     rsync remote:                                         OK (2.96s)
+2021-11-22T03:11:28.6641235Z     bup remote:                                           OK (0.94s)
+2021-11-22T03:11:28.6647373Z     borg remote:                                          OK
+2021-11-22T03:11:40.5680103Z     crypto:                                               OK (11.90s)
+2021-11-22T03:11:44.8147237Z     preferred content:                                    OK (4.24s)
+2021-11-22T03:11:47.0298718Z     required_content:                                     OK (2.22s)
+2021-11-22T03:11:49.3941918Z     add subdirs:                                          OK (2.36s)
+2021-11-22T03:11:50.8879458Z     addurl:                                               OK (1.50s)
+2021-11-22T03:11:50.8880438Z
+2021-11-22T03:11:50.8881162Z All 991 tests passed (978.21s)
+
+```
+
+
+### What version of git-annex are you using? On what operating system?
+
+8.20211118-g5a7f25397  linux ubuntu on github actions

diff --git a/doc/bugs/not_work__58___Git_annex_addurl_existing_file_without_/comment_2_a8ed9cc80a885e86d0acff6046b73f0c._comment b/doc/bugs/not_work__58___Git_annex_addurl_existing_file_without_/comment_2_a8ed9cc80a885e86d0acff6046b73f0c._comment
new file mode 100644
index 0000000000..a7777a0bfb
--- /dev/null
+++ b/doc/bugs/not_work__58___Git_annex_addurl_existing_file_without_/comment_2_a8ed9cc80a885e86d0acff6046b73f0c._comment
@@ -0,0 +1,84 @@
+[[!comment format=mdwn
+ username="m15"
+ avatar="http://cdn.libravatar.org/avatar/a8c5fb405daff5f9c0c15209a3df4a4d"
+ subject="new problem on reinject"
+ date="2021-11-23T00:53:37Z"
+ content="""
+Thanks for the suggestion!
+
+the cause:  `addunlocked=true`, maybe show better error message to help
+- if with `pidlock=true`, message is more brief, even harder to debug
+
+## version
+git-annex version
+```
+git-annex version: 8.20200226
+build flags: Assistant Webapp Pairing S3 WebDAV Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite
+dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed-1.0.1.0 ghc-8.6.5 http-client-0.5.14 persistent-sqlite-2.9.3 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs hook external
+operating system: linux x86_64
+supported repository versions: 8
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+local repository version: 8
+```
+
+## setting
+root/.gitattributes
+```
+* annex.largefiles=(largerthan=700kb)and(not((mimetype=text/*)or(include=*.html)))
+* annex.backend=SHA512E
+* annex.numcopies=2
+```
+
+disk partition type: ext4  (12GB spare space)
+
+### .git/config
+```
+[annex]
+    backend=SHA512E
+    addsmallfiles=true
+    addunlocked=true
+    genmetadata=true
+    alwayscommit=true
+    synccontent=false
+	youtube-dl-options = --no-overwrites ...
+    autocommit = false
+    resolvemerge = false
+[annex \"security\"]
+	# allowed-ip-addresses = all
+[filter \"annex\"]
+	smudge = git-annex smudge -- %f
+	clean = git-annex smudge --clean -- %f
+
+```
+## shell command
+with file `1.mp4` (size 3.1M) existing, run as shell-command in Python
+```
+git-annex addurl --file Move_Dance_20200806_203.m4a https://www.youtube.com/watch?v=VsgpUHUYuJI --relaxed && git-annex reinject 1.mp4 Move_Dance_20200806_203.m4a 
+```
+## output
+```
+addurl https://www.youtube.com/watch?v=VsgpUHUYuJI (to Move_Dance_20200806_203.m4a) ok
+(recording state in git...)
+reinject 1.mp4 ok
+(recording state in git...)
+  English/Move_Dance_20200806_203.m4a failed to generate a key
+git-annex: could not add file to the annex
+CallStack (from HasCallStack):
+  error, called at ./Command/Smudge.hs:141:24 in main:Command.Smudge
+error: external filter 'git-annex smudge --clean -- %f' failed 1
+error: external filter 'git-annex smudge --clean -- %f' failed
+  English/Move_Dance_20200806_203.m4a failed to generate a key
+git-annex: could not add file to the annex
+CallStack (from HasCallStack):
+  error, called at ./Command/Smudge.hs:141:24 in main:Command.Smudge
+error: external filter 'git-annex smudge --clean -- %f' failed 1
+error: external filter 'git-annex smudge --clean -- %f' failed
+```
+The result is a file staged for commit to git (not to git-annex).
+
+## Expected
+File added (reinjected) to git-annex as name `English/Move_Dance_20200806_203.m4a`
+
+"""]]

diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs
index 159d4f96ab..f4f95ff441 100644
--- a/Annex/AutoMerge.hs
+++ b/Annex/AutoMerge.hs
@@ -137,7 +137,7 @@ resolveMerge us them inoverlay = do
 	(fs, cleanup) <- inRepo (LsFiles.unmerged [top])
 	srcmap <- if inoverlay
 		then pure M.empty
-		else inodeMap $ pure (map LsFiles.unmergedFile fs, return True)
+		else inodeMap $ pure (concatMap getunmergedfiles fs, return True)
 	(mergedks, mergedfs) <- unzip <$> mapM (resolveMerge' srcmap us them inoverlay) fs
 	let mergedks' = concat mergedks
 	let mergedfs' = catMaybes mergedfs
@@ -160,6 +160,11 @@ resolveMerge us them inoverlay = do
 			cleanConflictCruft mergedks' mergedfs' unstagedmap
 		showLongNote "Merge conflict was automatically resolved; you may want to examine the result."
 	return merged
+  where
+	getunmergedfiles u = catMaybes
+		[ Just (LsFiles.unmergedFile u)
+		, LsFiles.unmergedSiblingFile u
+		]
 
 resolveMerge' :: InodeMap -> Maybe Git.Ref -> Git.Ref -> Bool -> LsFiles.Unmerged -> Annex ([Key], Maybe FilePath)
 resolveMerge' _ Nothing _ _ _ = return ([], Nothing)
@@ -177,7 +182,7 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 				unless inoverlay $
 					unless (islocked LsFiles.valUs) $
 						liftIO $ removeWhenExistsWith R.removeLink (toRawFilePath file)
-			| otherwise -> do
+			| otherwise -> resolveby [keyUs, keyThem] $
 				-- Only resolve using symlink when both
 				-- were locked, otherwise use unlocked
 				-- pointer.
@@ -185,13 +190,12 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 				if islocked LsFiles.valUs && islocked LsFiles.valThem
 					then makesymlink keyUs file
 					else makepointer keyUs file (combinedmodes)
-				return ([keyUs, keyThem], Just file)
 		-- Our side is annexed file, other side is not.
 		-- Make the annexed file into a variant file and graft in the
 		-- other file/directory as it was.
 		(Just keyUs, Nothing) -> resolveby [keyUs] $ do
 			graftin them file LsFiles.valThem LsFiles.valThem LsFiles.valUs
-			makevariantannexlink keyUs LsFiles.valUs 
+			makevariantannexlink keyUs LsFiles.valUs
 		-- Our side is not annexed file, other side is.
 		(Nothing, Just keyThem) -> resolveby [keyThem] $ do
 			graftin us file LsFiles.valUs LsFiles.valUs LsFiles.valThem
@@ -200,6 +204,7 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 		(Nothing, Nothing) -> return ([], Nothing)
   where
 	file = fromRawFilePath $ LsFiles.unmergedFile u
+	sibfile = fromRawFilePath <$> LsFiles.unmergedSiblingFile u
 
 	getkey select = 
 		case select (LsFiles.unmergedSha u) of
@@ -256,21 +261,27 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 	graftin b item selectwant selectwant' selectunwant = do
 		Annex.Queue.addUpdateIndex
 			=<< fromRepo (UpdateIndex.lsSubTree b item)
+				
+		let replacefile isexecutable = case selectwant' (LsFiles.unmergedSha u) of
+			Nothing -> noop
+			Just sha -> replaceWorkTreeFile item $ \tmp -> do
+				c <- catObject sha
+				liftIO $ L.writeFile tmp c
+				when isexecutable $
+					liftIO $ void $ tryIO $ 
+						modifyFileMode (toRawFilePath tmp) $
+							addModes executeModes
 
 		-- Update the work tree to reflect the graft.
 		unless inoverlay $ case (selectwant (LsFiles.unmergedTreeItemType u), selectunwant (LsFiles.unmergedTreeItemType u)) of
-			-- Symlinks are never left in work tree when
-			-- there's a conflict with anything else.
-			-- So, when grafting in a symlink, we must create it:
 			(Just TreeSymlink, _) -> do
 				case selectwant' (LsFiles.unmergedSha u) of
 					Nothing -> noop
 					Just sha -> do
 						link <- catSymLinkTarget sha
 						replacewithsymlink item link
-			-- And when grafting in anything else vs a symlink,
-			-- the work tree already contains what we want.
-			(_, Just TreeSymlink) -> noop
+			(Just TreeFile, Just TreeSymlink) -> replacefile False
+			(Just TreeExecutable, Just TreeSymlink) -> replacefile True
 			_ -> ifM (liftIO $ doesDirectoryExist item)
 				-- a conflict between a file and a directory
 				-- leaves the directory, so since a directory
@@ -278,23 +289,24 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 				( noop
 				-- probably a file with conflict markers is
 				-- in the work tree; replace with grafted
-				-- file content
-				, case selectwant' (LsFiles.unmergedSha u) of
-					Nothing -> noop
-					Just sha -> replaceWorkTreeFile item $ \tmp -> do
-						c <- catObject sha
-						liftIO $ L.writeFile tmp c
+				-- file content (this is needed when
+				-- the annexed file is unlocked)
+				, replacefile False
 				)
 	
 	resolveby ks a = do
-		{- Remove conflicted file from index so merge can be resolved. -}
+		{- Remove conflicted file from index so merge can be resolved.
+		 - If there's a sibling conflicted file, remove it too. -}
 		Annex.Queue.addCommand [] "rm"
 			[ Param "--quiet"
 			, Param "-f"
 			, Param "--cached"
 			, Param "--"
 			]
-			[file]
+			(catMaybes [Just file, sibfile])
+		liftIO $ maybe noop
+			(removeWhenExistsWith R.removeLink . toRawFilePath)
+			sibfile
 		void a
 		return (ks, Just file)
 
diff --git a/CHANGELOG b/CHANGELOG
index 57f98c424f..e67a08b44b 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,9 @@ git-annex (8.20211118) UNRELEASED; urgency=medium
   * Bugfix: When -J was enabled, getting files could leak an
     ever-growing number of git cat-file processes.
   * importfeed: Display url before starting youtube-dl download.
+  * Support git's new "ort" resolver, which became the default in git 2.34.0,
+    and broke the test suite and automatic merge resolution of a conflict
+    between an annexed file and a non-annexed file.
 
  -- Joey Hess <id@joeyh.name>  Wed, 17 Nov 2021 13:18:44 -0400
 
diff --git a/Git/LsFiles.hs b/Git/LsFiles.hs
index aff408deb6..42115de6e7 100644
--- a/Git/LsFiles.hs
+++ b/Git/LsFiles.hs
@@ -5,6 +5,8 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE OverloadedStrings #-}
+
 module Git.LsFiles (
 	Options(..),
 	inRepo,
@@ -236,7 +238,14 @@ data Unmerged = Unmerged
 	{ unmergedFile :: RawFilePath
 	, unmergedTreeItemType :: Conflicting TreeItemType
 	, unmergedSha :: Conflicting Sha
-	}
+	, unmergedSiblingFile :: Maybe RawFilePath
+	-- ^ Normally this is Nothing, because a
+	-- merge conflict is represented as a single file with two
+	-- stages. However, git resolvers sometimes choose to stage
+	-- two files, one for each side of the merge conflict. In such a case,
+	-- this is used for the name of the second file, which is related
+	-- to the first file. (Eg, "foo" and "foo~ref")
+	} deriving (Show)
 
 {- Returns a list of the files in the specified locations that have
  - unresolved merge conflicts.
@@ -246,7 +255,7 @@ data Unmerged = Unmerged
  -   1 = old version, can be ignored
  -   2 = us
  -   3 = them
- - If a line is omitted, that side removed the file.
+ - If line 2 or 3 is omitted, that side removed the file.
  -}
 unmerged :: [RawFilePath] -> Repo -> IO ([Unmerged], IO Bool)
 unmerged l repo = guardSafeForLsFiles repo $ do
@@ -265,7 +274,7 @@ data InternalUnmerged = InternalUnmerged
 	, ifile :: RawFilePath
 	, itreeitemtype :: Maybe TreeItemType
 	, isha :: Maybe Sha
-	}
+	} deriving (Show)
 
 parseUnmerged :: String -> Maybe InternalUnmerged
 parseUnmerged s
@@ -296,16 +305,25 @@ reduceUnmerged c (i:is) = reduceUnmerged (new:c) rest
 		{ unmergedFile = ifile i
 		, unmergedTreeItemType = Conflicting treeitemtypeA treeitemtypeB
 		, unmergedSha = Conflicting shaA shaB
+		, unmergedSiblingFile = if ifile sibi == ifile i
+			then Nothing
+			else Just (ifile sibi)
 		}
 	findsib templatei [] = ([], removed templatei)
 	findsib templatei (l:ls)
-		| ifile l == ifile templatei = (ls, l)
+		| ifile l == ifile templatei || issibfile templatei l = (ls, l)
 		| otherwise = (l:ls, removed templatei)
 	removed templatei = templatei
 		{ isus = not (isus templatei)

(Diff truncated)

diff --git a/Test.hs b/Test.hs
index 3d0f02ef89..6ec56b1b28 100644
--- a/Test.hs
+++ b/Test.hs
@@ -64,6 +64,7 @@ import qualified Annex.Init
 import qualified Annex.CatFile
 import qualified Annex.Path
 import qualified Annex.VectorClock
+import qualified Annex.VariantFile
 import qualified Annex.AdjustedBranch
 import qualified Annex.View
 import qualified Annex.View.ViewedFile
@@ -1369,12 +1370,11 @@ test_mixed_conflict_resolution = do
 			checkmerge "r2" r2
 	conflictor = "conflictor"
 	subfile = conflictor </> "subfile"
-	variantprefix = conflictor ++ ".variant"
 	checkmerge what d = do
 		doesDirectoryExist (d </> conflictor) 
 			@? (d ++ " conflictor directory missing")
 		l <- getDirectoryContents d
-		let v = filter (variantprefix `isPrefixOf`) l
+		let v = filter (Annex.VariantFile.variantMarker `isInfixOf`) l
 		not (null v)
 			@? (what ++ " conflictor variant file missing in: " ++ show l )
 		length v == 1
diff --git a/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_2_980556f1f7b339519593f9ead85ab854._comment b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_2_980556f1f7b339519593f9ead85ab854._comment
new file mode 100644
index 0000000000..00d768adaf
--- /dev/null
+++ b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_2_980556f1f7b339519593f9ead85ab854._comment
@@ -0,0 +1,34 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-11-22T16:32:10Z"
+ content="""
+The difference in behavior, when there's a conflict between a directory and a 
+file, is that the recursive strategy leaves the directory in "added by them"
+conflict state, as well as adding the file with a different name
+(and oddly, not staging it at all), while
+the ort strategy leaves does not flag the directory as in conflict,
+and makes the file be in "added by them" conflict state.
+
+`git-annex sync` resolves such a conflict adequately, with the new version
+of git, but the name of the file turns out different, eg 
+"foo~refs_remotes_origin_master.variant-b66a"
+or "foo~HEAD.variant-b66a" rather than
+"foo.variant-b66a". That happens only in such a mixed conflict,
+a conflict between two annexed files still gets the shorter name
+like "foo.variant-b66a".
+
+There's a small problem with a lack of a stable name being used, as shown
+above the name of the branch being merged from or to is used by git as part
+of the filename. So two people who resolve a merge this way would end up
+with two different names for the file. git-annex's merge conflict
+resolution is designed to yield a stable result no matter where it's run,
+so this behavior is a bit of a shame, but it's git's decision to behave
+that way, and I guess the same thing can happen when not using git-annex,
+if you commit what the new resolver stages. So this can be considered not a
+git-annex problem.
+
+I've adjusted the mixed conflict resolution test to accept these new names
+while also still working with the old names. That fixes that one. The other
+test is still failing.
+"""]]

diff --git a/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_1_184a259225e7cd14f85e77be4860abb6._comment b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_1_184a259225e7cd14f85e77be4860abb6._comment
new file mode 100644
index 0000000000..f3a345d469
--- /dev/null
+++ b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail/comment_1_184a259225e7cd14f85e77be4860abb6._comment
@@ -0,0 +1,86 @@
+[[!comment format=mdwn
+ username="kyle"
+ avatar="http://cdn.libravatar.org/avatar/7d6e85cde1422ad60607c87fa87c63f3"
+ subject="re: git 2.34: some conflict resolution unit tests fail"
+ date="2021-11-21T21:41:36Z"
+ content="""
+I see similar failures on GNU/Linux with Git 2.34.
+
+```
+$ git version
+git version 2.34.0
+$ git annex version | grep version:
+git-annex version: 8.20211118-ga0e9a059a
+
+$ git annex test -p 'conflict resolution '
+Tests
+  Unit Tests v8 adjusted unlocked branch
+    conflict resolution (adjusted branch):                Init Tests
+  init: OK (0.12s)
+  add:  OK (0.42s)
+
+All 2 tests passed (0.54s)
+OK (2.30s)
+    conflict resolution movein regression:                OK (2.18s)
+    conflict resolution (mixed directory and file):       FAIL (1.71s)
+      Test.hs:1378:
+      r1 conflictor variant file missing in: [\".\",\"..\",\"conflictor~HEAD.variant-cc12\",\"conflictor\",\"sha1foo\",\"foo\",\"bar.c\",\".git\"]
+      Use -p '/conflict resolution /&&/Unit Tests v8 adjusted unlocked branch.conflict resolution (mixed directory and file)/' to rerun this test only.
+    conflict resolution symlink bit:                      OK
+[... 206 lines ...]
+    conflict resolution (mixed locked and unlocked file): FAIL (1.51s)
+      Test.hs:1611:
+      r1 not exactly 0 variant files in: [\".\",\"..\",\"conflictor.variant-cc12\",\"conflictor~refs_remotes_r2_master.variant-cc12\",\"conflictor\",\"sha1foo\",\"foo\",\"bar.c\",\".git\"]
+      Use -p '/conflict resolution /&&/Unit Tests v8 locked.conflict resolution (mixed locked and unlocked file)/' to rerun this test only.
+
+8 out of 27 tests failed (53.21s)
+  (Failures above could be due to a bug in git-annex, or an incompatibility
+   with utilities, such as git, installed on this system.)
+```
+
+> Presumably this is due to the new ort merge strategy that has been
+> made default in this new release of git.
+
+Yes, that looks to be it.  The above failure goes away if I force
+git-annex to merge with the old recursive strategy:
+
+[[!format diff \"\"\"
+
+diff --git a/Git/Merge.hs b/Git/Merge.hs
+index b88d9a00f..c50672a3d 100644
+--- a/Git/Merge.hs
++++ b/Git/Merge.hs
+@@ -39,7 +39,8 @@ merge' extraparams branch mergeconfig commitmode r
+ 		go [Param $ fromRef branch]
+ 	| otherwise = go [Param \"--no-edit\", Param $ fromRef branch]
+   where
+-	go ps = merge'' (sp ++ [Param \"merge\"] ++ qp ++ ps ++ extraparams) mergeconfig r
++	go ps = merge'' (sp ++ [Param \"merge\"] ++ [Param \"-srecursive\"] ++
++			 qp ++ ps ++ extraparams) mergeconfig r
+ 	sp
+ 		| commitmode == AutomaticCommit =
+ 			[Param \"-c\", Param \"commit.gpgsign=false\"]
+
+\"\"\"]]
+
+```
+$ git version
+git version 2.34.0
+$ git annex version | grep version:
+git-annex version: 8.20211118-g30580a4e7
+
+$ git annex test -p 'conflict resolution '
+Tests
+  Unit Tests v8 adjusted unlocked branch
+    conflict resolution (adjusted branch):                Init Tests
+  init: OK (0.17s)
+  add:  OK (0.41s)
+
+All 2 tests passed (0.59s)
+OK (2.18s)
+    conflict resolution movein regression:                OK (2.24s)
+[... 37 lines ...]
+All 27 tests passed (67.12s)
+```
+
+"""]]

diff --git a/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail.mdwn b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail.mdwn
new file mode 100644
index 0000000000..0cf1b125fc
--- /dev/null
+++ b/doc/bugs/git_2.34__58___some_conflict_resolution_unit_tests_fail.mdwn
@@ -0,0 +1,139 @@
+### Please describe the problem.
+
+On my Windows laptop I upgraded to git 2.34 via Scoop and after that two conflict resolution unit tests
+started to fail. Presumably this is due to the new `ort` merge strategy that has been made default
+in this new release of git. (Ostensibly Recursive's Twin isn't perhaps quite yet what it claims to be.)
+
+### What steps will reproduce the problem?
+
+Install git version 2.34.0.windows.1. Then optionally build your git-annex with `stack setup && stack build`
+if you don't have a binary already. Then copy the binary to `C:\annxtmp1` and while situated in said directory
+in (Git) Bash say:
+```
+./git-annex test -p 'conflict resolution ' 2>&1 | tee git-annex..specific-test-01.LOG~202
+```
+Observe two out of nine tests failing.
+
+`test -p QuickCheck` was fine as were the other v8 unit tests.
+
+### What version of git-annex are you using? On what operating system?
+
+Two versions definitely exhibited this issue: 8.20211029-g9d3ce224e and 8.20211117-gc3af94eff (=released version).
+`git-annex version` output of the former below:
+
+[[!format sh """
+git-annex version: 8.20211029-g9d3ce224e
+build flags: Assistant Webapp Pairing TorrentParser Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: mingw32 x86_64
+supported repository versions: 8
+upgrade supported from repository versions: 2 3 4 5 6 7
+"""]]
+
+Windows version 21H1 (build 19043.1348), 64 bit.
+
+### Please provide any additional information below.
+
+_Under Windows with 8.20211029-g9d3ce224e:_
+
+[[!format sh """
+jkniiv@AINESIS MINGW64 /c/annxtmp1
+$ ./git-annex test -p 'conflict resolution ' 2>&1 | tee git-annex..specific-test-01.LOG~201
+[...]
+jkniiv@AINESIS MINGW64 /c/annxtmp1
+$ cat git-annex..specific-test-01.LOG~201
+Tests
+  Unit Tests v8 adjusted unlocked branch
+    conflict resolution (adjusted branch):                Init Tests
+  init: OK (5.64s)
+  add:  OK (7.85s)
+
+All 2 tests passed (13.49s)
+OK (46.59s)
+    conflict resolution movein regression:                OK (51.06s)
+    conflict resolution (mixed directory and file):       FAIL (37.66s)
+      Test.hs:1378:
+      r1 conflictor variant file missing in: ["sha1foo","foo","conflictor~HEAD.variant-cc12","conflictor","bar.c",".git","..","."]
+      Use -p '/conflict resolution /&&/conflict resolution (mixed directory and file)/' to rerun this test only.
+    conflict resolution symlink bit:                      OK
+    conflict resolution (uncommitted local file):         OK (53.99s)
+    conflict resolution (removed file):                   OK (106.99s)
+    conflict resolution (nonannexed file):                FAIL (26.74s)
+      .\\Test\\Framework.hs:57:
+      sync failed (transcript follows)
+      commit
+      On branch adjusted/master(unlocked)
+      nothing to commit, working tree clean
+      ok
+      pull r2
+      (Merging into master...)
+      CONFLICT (distinct types): conflictor had different types on each side; renamed one of them so each can be recorded somewhere.
+      Automatic merge failed; fix conflicts and then commit the result.
+      (recording state in git...)
+
+        Merge conflict was automatically resolved; you may want to examine the result.
+      U conflictor~refs_remotes_r2_master
+      (Merging into master...)
+      CONFLICT (distinct types): conflictor had different types on each side; renamed one of them so each can be recorded somewhere.
+      Automatic merge failed; fix conflicts and then commit the result.
+      (recording state in git...)
+
+        Merge conflict was automatically resolved; you may want to examine the result.
+      U conflictor~refs_remotes_r2_synced_master
+      failed
+      (merging r2/git-annex into git-annex...)
+      (recording state in git...)
+      push r2
+
+      failed
+      From ../../.t\tmprepo15
+       * [new branch]      adjusted/master(unlocked) -> r2/adjusted/master(unlocked)
+       * [new branch]      git-annex                 -> r2/git-annex
+       * [new branch]      master                    -> r2/master
+       * [new branch]      synced/master             -> r2/synced/master
+      error: Committing is not possible because you have unmerged files.
+      hint: Fix them up in the work tree, and then use 'git add/rm <file>'
+      hint: as appropriate to mark resolution and make a commit.
+      fatal: Exiting because of an unresolved conflict.
+      error: Committing is not possible because you have unmerged files.
+      hint: Fix them up in the work tree, and then use 'git add/rm <file>'
+      hint: as appropriate to mark resolution and make a commit.
+      fatal: Exiting because of an unresolved conflict.
+      To ../../.t\tmprepo15
+       * [new branch]      git-annex -> synced/git-annex
+       ! [rejected]        master -> synced/master (non-fast-forward)
+      error: failed to push some refs to '../../.t\tmprepo15'
+      hint: Updates were rejected because a pushed branch tip is behind its remote
+      hint: counterpart. Check out this branch and integrate the remote changes
+      hint: (e.g. 'git pull ...') before pushing again.
+      hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+      To ../../.t\tmprepo15
+       ! [rejected]        master -> master (non-fast-forward)
+      error: failed to push some refs to '../../.t\tmprepo15'
+      hint: Updates were rejected because a pushed branch tip is behind its remote
+      hint: counterpart. Check out this branch and integrate the remote changes
+      hint: (e.g. 'git pull ...') before pushing again.
+      hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+        Pushing to r2 failed.
+      sync: 2 failed
+
+      Use -p '/conflict resolution /&&/conflict resolution (nonannexed file)/' to rerun this test only.
+    conflict resolution (nonannexed symlink):             OK (22.07s)
+    conflict resolution (mixed locked and unlocked file): OK (41.50s)
+
+2 out of 9 tests failed (400.13s)
+  (Failures above could be due to a bug in git-annex, or an incompatibility
+   with utilities, such as git, installed on this system.)
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yeah sure, I'm a believer. :) Using it with my multigigabyte backup files just fine most of the time.
+I'm also building select versions of git-annex for both Windows native and WSL1/2 use following the
+development on master. A fine piece of software it definitely is.
+
+[[!meta author=jkniiv]]

diff --git a/doc/todo/adjusted_branch_with_hashdirlower_links.mdwn b/doc/todo/adjusted_branch_with_hashdirlower_links.mdwn
new file mode 100644
index 0000000000..9441ccf4d9
--- /dev/null
+++ b/doc/todo/adjusted_branch_with_hashdirlower_links.mdwn
@@ -0,0 +1,18 @@
+This would be like enabling the annex.tune.objecthashlower=true 
+[[tuning]] when creating a repository, but could be done after the fact,
+eg when a user needs to clone an existing reppsitory to a filesystem
+that has problems with mixed case hash directories.
+
+The adusted branch would simply convert annex links to use lower-case
+hashing, which is easy. But it would also need to enable storing new object
+files in lower-case hash directories, and move any currently stored object
+files to those directories.
+
+How to handle a `git checkout master` then? None of the annex links in
+master would work. Normally the user can leave an adjusted branch that way,
+but if they wanted to leave this one, there would have to be a command
+to move everything back.
+
+Note that, annexLocationsNonBare does fall back to looking for
+hashDirLower, so getting files from a repository with this enabled to
+another repository should work ok. --[[Joey]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_14_4bd86a227f117dbf5d1b12fbcf873e62._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_14_4bd86a227f117dbf5d1b12fbcf873e62._comment
new file mode 100644
index 0000000000..b444f382e5
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_14_4bd86a227f117dbf5d1b12fbcf873e62._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 14"
+ date="2021-11-19T17:44:22Z"
+ content="""
+  > fixed in [8.20211117-12-g623a77560](https://git.kitenet.net/index.cgi/git-annex.git/commit/?id=623a775609c486b5cd3441c5bca7c3d4a988eae0) --[[yarikoptic]]
+
+"""]]

diff --git a/Annex/Action.hs b/Annex/Action.hs
index c6f1c47583..95b440fe8c 100644
--- a/Annex/Action.hs
+++ b/Annex/Action.hs
@@ -13,7 +13,6 @@ module Annex.Action (
 	startup,
 	shutdown,
 	stopCoProcesses,
-	stopNonConcurrentSafeCoProcesses,
 ) where
 
 import qualified Data.Map as M
@@ -85,14 +84,8 @@ shutdown nocommit = do
 {- Stops all long-running child processes, including git query processes. -}
 stopCoProcesses :: Annex ()
 stopCoProcesses = do
-	stopNonConcurrentSafeCoProcesses
-	emptyTransferrerPool
-
-{- Stops long-running child processes that use handles that are not safe
- - for multiple threads to access at the same time. -}
-stopNonConcurrentSafeCoProcesses :: Annex ()
-stopNonConcurrentSafeCoProcesses = do
 	catFileStop
 	checkAttrStop
 	hashObjectStop
 	checkIgnoreStop
+	emptyTransferrerPool
diff --git a/Annex/Concurrent.hs b/Annex/Concurrent.hs
index 0851df8c91..2a9c20938d 100644
--- a/Annex/Concurrent.hs
+++ b/Annex/Concurrent.hs
@@ -101,14 +101,10 @@ dupState = do
 		, Annex.errcounter = 0
 		}
 
-{- Merges the passed AnnexState into the current Annex state.
- - Also closes various handles in it. -}
+{- Merges the passed AnnexState into the current Annex state. -}
 mergeState :: AnnexState -> Annex ()
 mergeState st = do
-	rd <- Annex.getRead id
-	st' <- liftIO $ (fst . snd)
-		<$> run (st, rd) stopNonConcurrentSafeCoProcesses
-	forM_ (M.toList $ Annex.cleanupactions st') $
+	forM_ (M.toList $ Annex.cleanupactions st) $
 		uncurry addCleanupAction
-	Annex.Queue.mergeFrom st'
-	changeState $ \s -> s { errcounter = errcounter s + errcounter st' }
+	Annex.Queue.mergeFrom st
+	changeState $ \s -> s { errcounter = errcounter s + errcounter st }
diff --git a/CHANGELOG b/CHANGELOG
index f8c27dd690..00b401c699 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,5 +1,7 @@
 git-annex (8.20211118) UNRELEASED; urgency=medium
 
+  * Bugfix: When -J was enabled, getting files leaked a ever-growing
+    number of git cat-file processes.
   * importfeed: Display url before starting youtube-dl download.
 
  -- Joey Hess <id@joeyh.name>  Wed, 17 Nov 2021 13:18:44 -0400
diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
index 944be373af..493f06b797 100644
--- a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
@@ -23,3 +23,5 @@ I'm unsure how to debug what is wrong, so seeking guidance.
 Tomorrow I'll start to see if I can reproduce with older/newer versions of git-annex.
 
 [[!tag forumbug]]
+
+[[bugs/done]]
diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_13_41e0fada90d795b9be12929ed66231e1._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_13_41e0fada90d795b9be12929ed66231e1._comment
new file mode 100644
index 0000000000..8df3c1700f
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_13_41e0fada90d795b9be12929ed66231e1._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 13"""
+ date="2021-11-19T16:22:47Z"
+ content="""
+Fixed.
+"""]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_89b9fe9c689515a8f874dee96864ff33._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_89b9fe9c689515a8f874dee96864ff33._comment
new file mode 100644
index 0000000000..6f91587616
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_89b9fe9c689515a8f874dee96864ff33._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2021-11-19T16:02:41Z"
+ content="""
+Reproduced the bug. Took laptop entirely offline, then `git-annex get -J5`
+in the repo @yoh provided above. Sometimes it happens immediately,
+sometimes it needs to wait a few minutes for DNS timeouts.
+
+Looks like one leaked git cat-file per time it failed to download in that
+situation.
+
+Should be able to take it from here..
+"""]]

diff --git a/Annex/Concurrent.hs b/Annex/Concurrent.hs
index fe8b7a8d3e..0851df8c91 100644
--- a/Annex/Concurrent.hs
+++ b/Annex/Concurrent.hs
@@ -17,6 +17,7 @@ import qualified Annex.Queue
 import Annex.Action
 import Types.Concurrency
 import Types.CatFileHandles
+import Annex.CatFile
 import Annex.CheckAttr
 import Annex.CheckIgnore
 
@@ -32,18 +33,31 @@ setConcurrency' NonConcurrent f =
 		{ Annex.concurrency = f NonConcurrent
 		}
 setConcurrency' c f = do
-	cfh <- getState Annex.catfilehandles
-	cfh' <- case cfh of
-		CatFileHandlesNonConcurrent _ -> liftIO catFileHandlesPool
-		CatFileHandlesPool _ -> pure cfh
-	cah <- mkConcurrentCheckAttrHandle c
-	cih <- mkConcurrentCheckIgnoreHandle c
-	Annex.changeState $ \s -> s
-		{ Annex.concurrency = f c
-		, Annex.catfilehandles = cfh'
-		, Annex.checkattrhandle = Just cah
-		, Annex.checkignorehandle = Just cih
-		}
+	oldc <- Annex.getState Annex.concurrency
+	case oldc of
+		ConcurrencyCmdLine NonConcurrent -> fromnonconcurrent
+		ConcurrencyGitConfig NonConcurrent -> fromnonconcurrent
+		_
+			| oldc == newc -> return ()
+			| otherwise ->
+				Annex.changeState $ \s -> s
+					{ Annex.concurrency = newc
+					}
+  where
+	newc = f c
+	fromnonconcurrent = do
+		catFileStop
+		checkAttrStop
+		checkIgnoreStop
+		cfh <- liftIO catFileHandlesPool
+		cah <- mkConcurrentCheckAttrHandle c
+		cih <- mkConcurrentCheckIgnoreHandle c
+		Annex.changeState $ \s -> s
+			{ Annex.concurrency = newc
+			, Annex.catfilehandles = cfh
+			, Annex.checkattrhandle = Just cah
+			, Annex.checkignorehandle = Just cih
+			}
 
 {- Allows forking off a thread that uses a copy of the current AnnexState
  - to run an Annex action.
diff --git a/Types/Concurrency.hs b/Types/Concurrency.hs
index ccab1ee0a9..9204f1d0f4 100644
--- a/Types/Concurrency.hs
+++ b/Types/Concurrency.hs
@@ -22,3 +22,4 @@ parseConcurrency s = Concurrent <$> readish s
 data ConcurrencySetting
 	= ConcurrencyCmdLine Concurrency
 	| ConcurrencyGitConfig Concurrency
+	deriving (Eq)
diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_b48f8d801305d9bde08060f06634b88e._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_b48f8d801305d9bde08060f06634b88e._comment
new file mode 100644
index 0000000000..c2df933601
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_12_b48f8d801305d9bde08060f06634b88e._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 12"""
+ date="2021-11-19T15:37:43Z"
+ content="""
+I found a case where zombie git processes could be started in theory, 
+but only when git-annex is run without -J. And only a few zombies I think.
+And I couldn't find a code path where it actually happened.
+So not the same as this bug. But it did involve setConcurrency, which 
+the bisected commit also involves (via forkState), so at least shows
+how that could cause a such a problem in theory. Fixed that.
+"""]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_10_3bc5af3b28094fc042c207e307fe0c18._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_10_3bc5af3b28094fc042c207e307fe0c18._comment
new file mode 100644
index 0000000000..c2b5191bd8
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_10_3bc5af3b28094fc042c207e307fe0c18._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 10"""
+ date="2021-11-19T13:30:25Z"
+ content="""
+I have not managed to reproduce it using that script. I let it run for 5
+minutes (and 1 gb transferred).
+
+Since that commit points to stall detection, I hacked the code to
+constantly detect false stalls. But I am still not seeing any zombie git
+processes with that.
+
+If stall detection *is* involved, I'd expect that you would see
+"Transfer seems to have stalled" when reproducing the bug.
+
+Since stall detection could depend on available bandwidth etc, I wonder if
+the script reproduces the bug reliably enough for the bisection to be
+correct. It would be helpful to manually verify the bisection result,
+with a longer test period than the script used. And look for the above
+message when reproducing it.
+"""]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
index 44956708dc..944be373af 100644
--- a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up.mdwn
@@ -22,4 +22,4 @@ I'm unsure how to debug what is wrong, so seeking guidance.
 
 Tomorrow I'll start to see if I can reproduce with older/newer versions of git-annex.
 
-
+[[!tag forumbug]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_9_b6780a99d7a175d2ff3e33e198c28a9d._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_9_b6780a99d7a175d2ff3e33e198c28a9d._comment
new file mode 100644
index 0000000000..8d13e7bf2a
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_9_b6780a99d7a175d2ff3e33e198c28a9d._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="bisection"
+ date="2021-11-18T02:29:01Z"
+ content="""
+if I bisected it right (didn't doublecheck manually), it is due to [8.20210127-107-gdd39e9e25 AKA 8.20210223~41](https://git.kitenet.net/index.cgi/git-annex.git/commit/?id=dd39e9e255a5684824ea75861f48f658eaaba288)
+
+<details>
+<summary>bisection script</summary> 
+
+```
+#!/bin/bash
+
+set -u
+
+u=http://datasets.datalad.org/labs/hasson/narratives/derivatives/fmriprep/.git
+d=/mnt/scrap/tmp/fmriprep
+
+[ -e \"$d\" ] || datalad clone \"$u\" \"$d\"
+cd \"$d\"
+git annex drop --all
+git annex get -J5 . >/dev/null &
+p=$!  # use pstree?
+for i in {1..200}; do # 200 sec seems to be enough
+    np=$(ps auxw | grep $USER | grep '\[git\] <defunct>' | wc -l)
+    echo -n \"$i $np \"
+    if [ $np -gt 10 ]; then { kill $p; exit 1; } fi
+    sleep 1
+done
+kill \"$p\" 
+echo done
+
+
+```
+</details>
+"""]]

diff --git a/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_8_02c5903bf4de4b94e6db84bc43b3b3d8._comment b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_8_02c5903bf4de4b94e6db84bc43b3b3d8._comment
new file mode 100644
index 0000000000..3d1d25abfa
--- /dev/null
+++ b/doc/forum/git-annex_causing_zombie_git_processes_to_build_up/comment_8_02c5903bf4de4b94e6db84bc43b3b3d8._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="god bless search"
+ date="2021-11-17T21:03:29Z"
+ content="""
+found this issue which was not `done`.
+
+I have ran into it again: on initial occasion I could not even ssh to that host because ~4000 `[git] <defunct>` processes forbid any other process to start according to the policy.
+Updated git-annex to standalone build from conda (8.20211012-geb95ed486), repeated again (on http://datasets.datalad.org/labs/hasson/narratives/derivatives/fmriprep/.git
+) with a plain `git annex get -J5 .` -- had to interrupt it when I saw that reached 1k of those.  Here is dump of annexy config variables so - no retries
+
+```
+(datalad) [d31548v@discovery7 fmriprep]$ git config --list | grep annex
+annex.pidlock=true
+remote.origin.annex-bare=false
+remote.origin.annex-uuid=348d8bd8-1142-4de1-b52e-6b9899faa1d5
+annex.uuid=75d12ed4-e3e4-4ad0-ba7e-ec11538966d2
+annex.version=8
+filter.annex.smudge=git-annex smudge -- %f
+filter.annex.clean=git-annex smudge --clean -- %f
+remote.fcp-indi.annex-s3=true
+remote.fcp-indi.annex-uuid=bf6f56f3-f6b7-4df8-9898-a4226dc71400
+```
+
+I wish Nick_P has shared the script -- I could try to bisect... otherwise someone needs to write one ;-)
+"""]]

diff --git a/doc/news/version_8.20210714.mdwn b/doc/news/version_8.20210714.mdwn
deleted file mode 100644
index c66eba6141..0000000000
--- a/doc/news/version_8.20210714.mdwn
+++ /dev/null
@@ -1,12 +0,0 @@
-git-annex 8.20210714 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * assistant: Avoid unncessary git repository repair in a situation where
-    git fsck gets confused about a commit that is made while it's running.
-  * addurl: Avoid crashing when used on beegfs.
-  * --debug output goes to stderr again, not stdout.
-    (Reversion in version 8.20210428)
-  * init: Fix misbehavior when core.sharedRepository = group that
-    caused it to enter an adjusted branch and set annex.crippledfilesystem
-    (Reversion in version 8.20210630)
-  * assistant: When adding non-large files to git, honor annex.delayadd
-    configuration. Also, don't add non-large files to git when they
-    are still being written to."""]]
\ No newline at end of file
diff --git a/doc/news/version_8.20211117.mdwn b/doc/news/version_8.20211117.mdwn
new file mode 100644
index 0000000000..94ce39323f
--- /dev/null
+++ b/doc/news/version_8.20211117.mdwn
@@ -0,0 +1,18 @@
+git-annex 8.20211117 released with [[!toggle text="these changes"]]
+[[!toggleable text="""  * filter-process: New command that can make git add/checkout faster when
+    there are a lot of unlocked annexed files or non-annexed files, but that
+    also makes git add of large annexed files slower. Use it by running:
+    git config filter.annex.process 'git-annex filter-process'
+  * Fix a typo in the name of youtube-dl
+    (reversion introduced in version 8.20210903)
+  * git-lfs: Fix interoperability with gitlab's implementation of the
+    git-lfs protocol, which requests Content-Encoding chunked.
+  * importfeed: Fix a crash when used in a non-unicode locale.
+  * migrate: New --remove-size option.
+  * uninit: Avoid error message when no commits have been made to the
+    repository yet.
+  * uninit: Avoid error message when there is no git-annex branch.
+  * metadata --batch: Avoid crashing when a non-annexed file is input,
+    instead output a blank line like other batch commands do.
+  * metadata --batch --json: Reject input whose "fields" does not consist
+    of arrays of strings. Such invalid input used to be silently ignored."""]]
\ No newline at end of file

diff --git a/doc/projects/dandi.mdwn b/doc/projects/dandi.mdwn
index d485a27fc8..5ab5738171 100644
--- a/doc/projects/dandi.mdwn
+++ b/doc/projects/dandi.mdwn
@@ -7,19 +7,20 @@ DANDI: Distributed Archives for Neurophysiology Data Integration is a platform f
 
 [[!inline pages="todo/* and !todo/done and !link(todo/done) and tagged(projects/dandi)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-### Done:
+<details>
+<summary>Done</summary>
 
 [[!inline pages="todo/* and !todo/done and link(todo/done) and tagged(projects/dandi)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-
+</details>
 
 ## BUGs
 
 [[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and tagged(projects/dandi)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-### Done:
+<details>
+<summary>Done</summary>
 
 [[!inline pages="bugs/* and !bugs/done and link(bugs/done) and tagged(projects/dandi)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-
-
+</details>
diff --git a/doc/projects/repronim.mdwn b/doc/projects/repronim.mdwn
index e31221ce87..e309c8db8f 100644
--- a/doc/projects/repronim.mdwn
+++ b/doc/projects/repronim.mdwn
@@ -7,16 +7,20 @@ The center for Reproducible Neuroimaging computation develops standards and tool
 
 [[!inline pages="todo/* and !todo/done and !link(todo/done) and tagged(projects/repronim)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-### Done:
+<details>
+<summary>Done</summary>
 
 [[!inline pages="todo/* and !todo/done and link(todo/done) and tagged(projects/repronim)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-
+</details>
 
 ## BUGs
 
 [[!inline pages="bugs/* and !bugs/done and !link(bugs/done) and tagged(projects/repronim)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
 
-### Done:
+<details>
+<summary>Done</summary>
 
 [[!inline pages="bugs/* and !bugs/done and link(bugs/done) and tagged(projects/repronim)" sort=mtime feeds=no actions=yes archive=yes show=0 template=buglist]] 
+
+</details>

diff --git a/doc/bugs/gpgconf__58___invalid_option___34__--kill__34_____40__gpg_2.0.22__41___.mdwn b/doc/bugs/gpgconf__58___invalid_option___34__--kill__34_____40__gpg_2.0.22__41___.mdwn
index bab7443d90..ada993ecd0 100644
--- a/doc/bugs/gpgconf__58___invalid_option___34__--kill__34_____40__gpg_2.0.22__41___.mdwn
+++ b/doc/bugs/gpgconf__58___invalid_option___34__--kill__34_____40__gpg_2.0.22__41___.mdwn
@@ -31,3 +31,5 @@ Meanwhile - adding `gnupg >=2.1.1 ` dependency to conda build: https://github.co
 
 [[!meta author=yoh]]
 [[!tag projects/repronim]]
+
+[[!tag moreinfo]]

diff --git a/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_17_7db34973cb362bee271545c3c58690b4._comment b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_17_7db34973cb362bee271545c3c58690b4._comment
new file mode 100644
index 0000000000..62774e4b12
--- /dev/null
+++ b/doc/bugs/Windows__58___substantial_per-file_cost_for___96__add__96__/comment_17_7db34973cb362bee271545c3c58690b4._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 17"""
+ date="2021-11-16T16:28:11Z"
+ content="""
+The new `git-annex filter-process` should improve this speed a lot.
+It avoids a new process being started for each file that is added.
+
+git config filter.annex.process 'git-annex filter-process'
+
+That may become the default in v9, or possibly in new v8 repositories.
+
+There is a tradeoff, since `git add` of a large file to the annex gets
+slower when it's enabled. Only about 5% in my benchmarks on linux, but
+maybe more on windows, I don't know.
+"""]]

diff --git a/doc/bugs/fsck_always_fails.mdwn b/doc/bugs/fsck_always_fails.mdwn
index e225e8b77f..9d301324fa 100644
--- a/doc/bugs/fsck_always_fails.mdwn
+++ b/doc/bugs/fsck_always_fails.mdwn
@@ -48,3 +48,7 @@ I was using some older version from 2020 previously
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 Yes, I've been using it for years. It's a robust and reliable piece of software. :)
+
+> This output does not seem unclear. The local repository being fscked is
+> marked as dead, and of course git-annex treats dead repositories as not
+> containing any content. [[done]] --[[Joey]]

diff --git a/CHANGELOG b/CHANGELOG
index 933804b64c..2846d0aaaf 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -16,6 +16,7 @@ git-annex (8.20211029) UNRELEASED; urgency=medium
   * migrate: New --remove-size option.
   * Fix a typo in the name of youtube-dl
     (reversion introduced in version 8.20210903)
+  * importfeed: Fix a crash when used in a non-unicode locale.
 
  -- Joey Hess <id@joeyh.name>  Mon, 01 Nov 2021 13:19:46 -0400
 
diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index a8d21365e3..922e354c21 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -20,7 +20,9 @@ import Data.Time.Format
 import Data.Time.Calendar
 import Data.Time.LocalTime
 import qualified Data.Text as T
+import qualified Data.Text.Encoding as TE
 import qualified System.FilePath.ByteString as P
+import qualified Data.ByteString as B
 
 import Command
 import qualified Annex
@@ -161,10 +163,10 @@ findDownloads u f = catMaybes $ map mk (feedItems f)
 	mk i = case getItemEnclosure i of
 		Just (enclosureurl, _, _) ->
 			Just $ ToDownload f u i $ Enclosure $ 
-				T.unpack enclosureurl
+				decodeBS $ fromFeedText enclosureurl
 		Nothing -> case getItemLink i of
 			Just l -> Just $ ToDownload f u i $ 
-				MediaLink $ T.unpack l
+				MediaLink $ decodeBS $ fromFeedText l
 			Nothing -> Nothing
 
 {- Feeds change, so a feed download cannot be resumed. -}
@@ -243,7 +245,7 @@ performDownload addunlockedmatcher opts cache todownload = case location todownl
 
 	knownitemid = case getItemId (item todownload) of
 		Just (_, itemid) ->
-			S.member (T.unpack itemid) (knownitems cache)
+			S.member (decodeBS $ fromFeedText itemid) (knownitems cache)
 		_ -> False
 
 	rundownload url extension getter = do
@@ -370,7 +372,7 @@ feedFile tmpl i extension = sanitizeLeadingFilePathCharacter $
 		Just pd -> Just $
 			formatTime defaultTimeLocale "%F" pd
 		-- if date cannot be parsed, use the raw string
-		Nothing-> replace "/" "-" . T.unpack
+		Nothing-> replace "/" "-" . decodeBS . fromFeedText
 			<$> getItemPublishDateString itm
 	
 	(itempubyear, itempubmonth, itempubday) = case pubdate of
@@ -401,7 +403,7 @@ minimalMetaData :: ToDownload -> MetaData
 minimalMetaData i = case getItemId (item i) of
 	(Nothing) -> emptyMetaData
 	(Just (_, itemid)) -> MetaData $ M.singleton itemIdField 
-		(S.singleton $ toMetaValue $ encodeBS $ T.unpack itemid)
+		(S.singleton $ toMetaValue $fromFeedText itemid)
 
 {- Extract fields from the feed and item, that are both used as metadata,
  - and to generate the filename. -}
@@ -411,18 +413,18 @@ extractFields i = map (uncurry extractField)
 	, ("itemtitle", [itemtitle])
 	, ("feedauthor", [feedauthor])
 	, ("itemauthor", [itemauthor])
-	, ("itemsummary", [T.unpack <$> getItemSummary (item i)])
-	, ("itemdescription", [T.unpack <$> getItemDescription (item i)])
-	, ("itemrights", [T.unpack <$> getItemRights (item i)])
-	, ("itemid", [T.unpack . snd <$> getItemId (item i)])
+	, ("itemsummary", [decodeBS . fromFeedText <$> getItemSummary (item i)])
+	, ("itemdescription", [decodeBS . fromFeedText <$> getItemDescription (item i)])
+	, ("itemrights", [decodeBS . fromFeedText <$> getItemRights (item i)])
+	, ("itemid", [decodeBS . fromFeedText . snd <$> getItemId (item i)])
 	, ("title", [itemtitle, feedtitle])
 	, ("author", [itemauthor, feedauthor])
 	]
   where
-	feedtitle = Just $ T.unpack $ getFeedTitle $ feed i
-	itemtitle = T.unpack <$> getItemTitle (item i)
-	feedauthor = T.unpack <$> getFeedAuthor (feed i)
-	itemauthor = T.unpack <$> getItemAuthor (item i)
+	feedtitle = Just $ decodeBS $ fromFeedText $ getFeedTitle $ feed i
+	itemtitle = decodeBS . fromFeedText <$> getItemTitle (item i)
+	feedauthor = decodeBS . fromFeedText <$> getFeedAuthor (feed i)
+	itemauthor = decodeBS . fromFeedText <$> getItemAuthor (item i)
 
 itemIdField :: MetaField
 itemIdField = mkMetaFieldUnchecked "itemid"
@@ -478,3 +480,17 @@ clearFeedProblem url =
 
 feedState :: URLString -> Annex RawFilePath
 feedState url = fromRepo $ gitAnnexFeedState $ fromUrl url Nothing
+
+{- The feed library parses the feed to Text, and does not use the
+ - filesystem encoding to do it, so when the locale is not unicode
+ - capable, a Text value can still include unicode characters. 
+ -
+ - So, it's not safe to use T.unpack to convert that to a String,
+ - because later use of that String by eg encodeBS will crash
+ - with an encoding error. Use this instad.
+ -
+ - This should not be used on a Text that is read using the
+ - filesystem encoding because it does not reverse that encoding.
+ -}
+fromFeedText :: T.Text -> B.ByteString
+fromFeedText = TE.encodeUtf8
diff --git a/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn
index e7f7d445ee..007ad9b806 100644
--- a/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn
+++ b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn
@@ -69,3 +69,5 @@ $ echo $?
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 
 I've been using git annex for a few years now primarily as a podcatcher. I've also been using it to manage my ebooks and devices. I'm slowly starting to use it for managing my personal (non-text) records and documents.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument/comment_1_a36feb24e627bba9c146fc791fca2a6b._comment b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument/comment_1_a36feb24e627bba9c146fc791fca2a6b._comment
new file mode 100644
index 0000000000..673c8ec88e
--- /dev/null
+++ b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument/comment_1_a36feb24e627bba9c146fc791fca2a6b._comment
@@ -0,0 +1,60 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-15T16:04:31Z"
+ content="""
+To reproduce this, I had to set LANG=C. Using a unicode locale avoids
+the problem.
+
+The `.met` file indicates it's a problem with encoding of metadata
+that is being imported from the feed, and so it must be the
+itemid that is causing the problem.
+
+        <item>
+		...
+                <guid>http://www.manager-tools.com/2014/01/choosing-a-company-work-chapter-7-–-questions/</guid>
+        </item>
+
+A file with the feed edited down to just that item is enough to reproduce it.
+Notice the unicode in the guid "chapter-7-–-questions".
+That ENDASH character is causing the crash.
+
+Also I noticed that the next time it runs, it skips the item, since it got
+far enough to add the file for it and record the url before the metadata itemid
+write crashed it. Explains why it's failing on different items in different runs.
+
+While this looks like one of the old Handle output encoding problems, it is not,
+because a) the itemid is written as a ByteString so encoding does not matter,
+b) those were fixed comprehensively by forcing all handles to use filesystem
+enconding, and c) just printing out the length of the itemid also causes a crash:
+
+	+                 liftIO $ print (L.length (journalableByteString content))
+
+	git-annex: recoverEncode: invalid argument (invalid character)
+
+Looking at what the feed library parses:
+	
+	LANG=C ghci Utility/FileSystemEncoding.hs
+	ghci> Just f <- Text.Feed.Import.parseFeedFromFile "career_tools_podcasts.xml"
+	ghci> Just (_, x) = Text.Feed.Query.getItemId (Text.Feed.Query.feedItems f !! 0)
+	ghci> x
+	"http://www.manager-tools.com/2014/01/choosing-a-company-work-chapter-7-\8211-questions/"
+	ghci> encodeBS (Data.Text.unpack x)
+	"*** Exception: recoverEncode: invalid argument (invalid character)
+
+So the problem is that Text parses the feed as unicode, leading to this
+non-ascii Char that is not encoded using the filesystem encoding
+(which would encode it as "\56546\56448\56467"). 
+And `encodeBS "\8211"` crashes in LANG=C.
+
+Which is a reversion of sorts; before [[!commit fa62c98910]] encodeBS did
+not crash. Although it also didn't round-trip this value properly,
+producing "M" for it. Since this only affects strings that are not input in
+the filesystem encoding, I think the new encodeBS is still ok to use
+generally; I'm not going to revert that commit.
+
+Instead, Text values originating from Feed need to be converted to
+String in some other way, producing a value encoded
+using the filesystem encoding. encodeUtf8 looks like it will
+do the right thing in this case.
+"""]]

diff --git a/doc/bugs/fsck_always_fails/comment_4_81ccdf94101f1080521d36aa90a376ac._comment b/doc/bugs/fsck_always_fails/comment_4_81ccdf94101f1080521d36aa90a376ac._comment
new file mode 100644
index 0000000000..8126445061
--- /dev/null
+++ b/doc/bugs/fsck_always_fails/comment_4_81ccdf94101f1080521d36aa90a376ac._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Lukey"
+ avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b"
+ subject="comment 4"
+ date="2021-11-14T20:11:35Z"
+ content="""
+Well, you somehow marked the repo you're working in as dead. Usually you only mark a repo that is irretrievably lost (due to drive failure for example) as dead so git-annex knows that none of the data in it exists anymore and doesn't count it as a copy anymore. That's exactly what you're seeing here.
+"""]]

diff --git a/doc/bugs/fsck_always_fails/comment_3_aa0e5955253e94863479d66aedd9d430._comment b/doc/bugs/fsck_always_fails/comment_3_aa0e5955253e94863479d66aedd9d430._comment
new file mode 100644
index 0000000000..909fc888dd
--- /dev/null
+++ b/doc/bugs/fsck_always_fails/comment_3_aa0e5955253e94863479d66aedd9d430._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="contact@ee563aaec1e9de7a4e8d748992963dba79178e9c"
+ nickname="contact"
+ avatar="http://cdn.libravatar.org/avatar/4aa4d7a441bd4e6948fc0012c7229d01"
+ subject="comment 3"
+ date="2021-11-14T18:41:18Z"
+ content="""
+Sorry for the comment spam. It looks like 'Warning: Fscking a repository that is currently marked as dead.' is what was happening here.
+The error message is confusing making it seem like the checksum failed and that the file existed elsewhere, not that there's no copies.
+"""]]

diff --git a/doc/bugs/fsck_always_fails/comment_2_b56c7aaf88535830a9f153bba27ececd._comment b/doc/bugs/fsck_always_fails/comment_2_b56c7aaf88535830a9f153bba27ececd._comment
new file mode 100644
index 0000000000..42ac2a6230
--- /dev/null
+++ b/doc/bugs/fsck_always_fails/comment_2_b56c7aaf88535830a9f153bba27ececd._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="contact@ee563aaec1e9de7a4e8d748992963dba79178e9c"
+ nickname="contact"
+ avatar="http://cdn.libravatar.org/avatar/4aa4d7a441bd4e6948fc0012c7229d01"
+ subject="comment 2"
+ date="2021-11-14T18:37:04Z"
+ content="""
+Uh, addendum 2: I signed in with my email which is contact@jookia.org , but it's created a user named 'contact'. Sorry for eating up this username. :)
+"""]]

diff --git a/doc/bugs/fsck_always_fails/comment_1_d0620c59d6ada267792d0cde7b3c43f3._comment b/doc/bugs/fsck_always_fails/comment_1_d0620c59d6ada267792d0cde7b3c43f3._comment
new file mode 100644
index 0000000000..bb468c553a
--- /dev/null
+++ b/doc/bugs/fsck_always_fails/comment_1_d0620c59d6ada267792d0cde7b3c43f3._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="contact@ee563aaec1e9de7a4e8d748992963dba79178e9c"
+ nickname="contact"
+ avatar="http://cdn.libravatar.org/avatar/4aa4d7a441bd4e6948fc0012c7229d01"
+ subject="comment 1"
+ date="2021-11-14T18:35:45Z"
+ content="""
+I'm confused as to why it says random data is in an old repository.
+numcopies 0 seems to 'fix' this?
+"""]]

diff --git a/doc/bugs/fsck_always_fails.mdwn b/doc/bugs/fsck_always_fails.mdwn
new file mode 100644
index 0000000000..e225e8b77f
--- /dev/null
+++ b/doc/bugs/fsck_always_fails.mdwn
@@ -0,0 +1,50 @@
+### Please describe the problem.
+
+All my files added to my archive repo over the past few months seem to fail git annex fsck.
+
+### What steps will reproduce the problem?
+
+- Add a file to the repo
+- Fsck it
+
+### What version of git-annex are you using? On what operating system?
+
+8.20211012-geb95ed486 on Arch Linux directly downloaded from this website
+I was using some older version from 2020 previously
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+    jookia@titan:/data/jookia/archive% head -c1024 /dev/urandom > RANDOM.TEST                          (exited 130) (1 jobs) 05:27
+    jookia@titan:/data/jookia/archive% git annex add RANDOM.TEST                                                    (1 jobs) 05:27
+    add RANDOM.TEST 
+    ok                                
+    (recording state in git...)
+    jookia@titan:/data/jookia/archive% git commit -m RANDOM.TEST                                                    (1 jobs) 05:27
+    [main 2e9f2429018] RANDOM.TEST
+     1 file changed, 1 insertion(+)
+     create mode 120000 RANDOM.TEST
+    jookia@titan:/data/jookia/archive% git annex fsck RANDOM.TEST                                                   (1 jobs) 05:27
+
+      Remote black cannot currently be accessed.
+    Warning: Fscking a repository that is currently marked as dead.
+    fsck RANDOM.TEST (checksum...) 
+      ** No known copies exist of RANDOM.TEST
+      These dead repositories used to have copies
+            e62ad85d-368d-4ea5-a85c-e94598904a50 -- T400 laptop [here]
+    failed
+    (recording state in git...)
+    fsck: 1 failed
+    jookia@titan:/data/jookia/archive% ls -l RANDOM.TEST                                               (exited 130) (1 jobs) 05:28
+    lrwxrwxrwx 1 jookia jookia 194 Nov 15 05:27 RANDOM.TEST -> .git/annex/objects/K4/jV/SHA256E-s1024--0dedc9e7fb3fa053982201c2166fc1265ebd64d41ab91fe1c348c2df46f50167.TEST/SHA256E-s1024--0dedc9e7fb3fa053982201c2166fc1265ebd64d41ab91fe1c348c2df46f50167.TEST
+    jookia@titan:/data/jookia/archive% sha256sum RANDOM.TEST                                                        (1 jobs) 05:28
+    0dedc9e7fb3fa053982201c2166fc1265ebd64d41ab91fe1c348c2df46f50167  RANDOM.TEST
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, I've been using it for years. It's a robust and reliable piece of software. :)

diff --git a/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn
new file mode 100644
index 0000000000..e7f7d445ee
--- /dev/null
+++ b/doc/bugs/importfeed_failing_with_hPut__58___invalid_argument.mdwn
@@ -0,0 +1,71 @@
+### Please describe the problem.
+
+When I use importfeed to get my podcasts after an update, it's starts to fail on some items and the metadata doesn't get generated. The error given is hPut: invalid argument (invalid character).
+
+### What steps will reproduce the problem?
+
+1. importfeed of a previously working podcast feed
+
+I also noticed that this tends to happen more often when using genmetadata but it still happens when that's not used.
+
+### What version of git-annex are you using? On what operating system?
+
+I'm on Ubuntu 20.04 (just upgraded from 18.04) with git-annex installed from nix.
+
+    git-annex version: 8.20211011
+    build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+    dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.6.4.1 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 
+    BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 
+    BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+    operating system: linux x86_64
+    supported repository versions: 8
+    upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+    local repository version: 8
+
+I've had the same problem on my Synology NAS:
+
+    git-annex version: 8.20211012-geb95ed486
+    build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+    dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+    operating system: linux x86_64
+    supported repository versions: 8
+    upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+
+
+This is a relatively recent issue for me. I (maybe foolishly) updated my laptop and synology versions at the same time. The NAS was on git-annex-standalone-amd64-7.20191115. I was able to rollback nix to find the last working version and it seems it was:
+
+    git-annex version: 8.20210330
+    build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+    dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.28 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.4 http-client-0.6.4.1 persistent-sqlite-2.11.1.0 torrent-10000.1.1 uuid-1.3.14 yesod-1.6.1.0
+    key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256     BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+    remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+    operating system: linux x86_64
+    supported repository versions: 8
+    upgrade supported from repository versions: 0 1 2 3 4 5 6 7
+    local repository version: 8
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+$ git init -b main test && cd test && git annex init && git config annex.genmetadata true
+$ git-annex importfeed --template='new/${feedtitle}/${itempubdate}-${itemtitle}${extension}' --relaxed --fast https://files.manager-tools.com/files/public/feeds/career_tools_podcasts.xml
+[...]
+addurl https://traffic.libsyn.com/secure/manager-tools/career-tools-2020-12-24.mp3 (to new/Career_Tools/2020-12-24-MT_Goals_Examples_-_HOF_2020.mp3) git-annex: .git/annex/othertmp/a42_240_URL-s18703819--https&c%%traffic.libsyn.com%secu-da948bd01d45574da2155e03220b1a96.log.met: hPut: invalid argument (invalid character)
+
+$ echo $?
+1
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I've been using git annex for a few years now primarily as a podcatcher. I've also been using it to manage my ebooks and devices. I'm slowly starting to use it for managing my personal (non-text) records and documents.

diff --git a/doc/git-annex-adjust.mdwn b/doc/git-annex-adjust.mdwn
index 644e9fb267..42ec030a20 100644
--- a/doc/git-annex-adjust.mdwn
+++ b/doc/git-annex-adjust.mdwn
@@ -16,13 +16,18 @@ The adjusted branch will have a name like "adjusted/master(unlocked)".
 Since it's a regular git branch, you can use `git checkout` to switch
 back to the original branch at any time.
 
+This allows changing how annexed files are handled, without making changes
+to a public branch with commands like `git-annex unlock`.
+
 While in the adjusted branch, you can use git-annex and git commands as
 usual. Any commits that you make will initially only be made to the
 adjusted branch. 
 
 To propagate commits from the adjusted branch back to the original branch,
 and to other repositories, as well as to merge in changes from other
-repositories, run `git annex sync`.
+repositories, run `git annex sync`. This will propagate changes that you've
+made such as adding/deleting files, but will not propagate the adjustments
+made by this command.
 
 When in an adjusted branch, using `git merge otherbranch` is often not
 ideal, because merging a non-adjusted branch may lead to unncessary
@@ -117,6 +122,8 @@ and will also propagate commits back to the original branch.
 
 [[git-annex-unlock]](1)
 
+[[git-annex-lock]](1)
+
 [[git-annex-upgrade]](1)
 
 [[git-annex-sync]](1)

diff --git a/CHANGELOG b/CHANGELOG
index 6474b20204..0b9ff1bb6f 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -13,6 +13,7 @@ git-annex (8.20211029) UNRELEASED; urgency=medium
   * uninit: Avoid error message when there is no git-annex branch.
   * git-lfs: Fix interoperability with gitlab's implementation of the
     git-lfs protocol, which requests Content-Encoding chunked.
+  * migrate: New --remove-size option.
 
  -- Joey Hess <id@joeyh.name>  Mon, 01 Nov 2021 13:19:46 -0400
 
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 9a0d69f35a..1844f9a63b 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -23,20 +23,33 @@ cmd :: Command
 cmd = withGlobalOptions [annexedMatchingOptions] $
 	command "migrate" SectionUtility 
 		"switch data to different backend"
-		paramPaths (withParams seek)
+		paramPaths (seek <$$> optParser)
 
-seek :: CmdParams -> CommandSeek
-seek = withFilesInGitAnnex ww seeker <=< workTreeItems ww
+data MigrateOptions = MigrateOptions
+	{ migrateThese :: CmdParams
+	, removeSize :: Bool
+	}
+
+optParser :: CmdParamsDesc -> Parser MigrateOptions
+optParser desc = MigrateOptions
+	<$> cmdParams desc
+	<*> switch
+		( long "remove-size"
+		<> help "remove size field from keys"
+		)
+
+seek :: MigrateOptions -> CommandSeek
+seek o = withFilesInGitAnnex ww seeker =<< workTreeItems ww (migrateThese o)
   where
 	ww = WarnUnmatchLsFiles
 	seeker = AnnexedFileSeeker
-		{ startAction = start
+		{ startAction = start o
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
 
-start :: SeekInput -> RawFilePath -> Key -> CommandStart
-start si file key = do
+start :: MigrateOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
+start o si file key = do
 	forced <- Annex.getState Annex.force
 	v <- Backend.getBackend (fromRawFilePath file) key
 	case v of
@@ -46,9 +59,14 @@ start si file key = do
 			newbackend <- maybe defaultBackend return 
 				=<< chooseBackend file
 			if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists
-				then starting "migrate" (mkActionItem (key, file)) si $
-					perform file key oldbackend newbackend
-				else stop
+				then go False oldbackend newbackend
+				else if removeSize o && exists
+					then go True oldbackend oldbackend
+					else stop
+  where
+	go onlyremovesize oldbackend newbackend =
+		starting "migrate" (mkActionItem (key, file)) si $
+			perform onlyremovesize o file key oldbackend newbackend
 
 {- Checks if a key is upgradable to a newer representation.
  - 
@@ -70,13 +88,14 @@ upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
  - data cannot get corrupted after the fsck but before the new key is
  - generated.
  -}
-perform :: RawFilePath -> Key -> Backend -> Backend -> CommandPerform
-perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
+perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> Backend -> Backend -> CommandPerform
+perform onlyremovesize o file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
   where
 	go Nothing = stop
 	go (Just (newkey, knowngoodcontent))
-		| knowngoodcontent = finish newkey
-		| otherwise = stopUnless checkcontent $ finish newkey
+		| knowngoodcontent = finish (removesize newkey)
+		| otherwise = stopUnless checkcontent $
+			finish (removesize newkey)
 	checkcontent = Command.Fsck.checkBackend oldbackend oldkey Command.Fsck.KeyPresent afile
 	finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey)
 		( do
@@ -89,6 +108,7 @@ perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbacken
 			next $ Command.ReKey.cleanup file newkey
 		, giveup "failed creating link from old to new key"
 		)
+	genkey _ | onlyremovesize = return $ Just (oldkey, False)
 	genkey Nothing = do
 		content <- calcRepo $ gitAnnexLocation oldkey
 		let source = KeySource
@@ -101,4 +121,7 @@ perform file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbacken
 	genkey (Just fm) = fm oldkey newbackend afile >>= \case
 		Just newkey -> return (Just (newkey, True))
 		Nothing -> genkey Nothing
+	removesize k
+		| removeSize o = alterKey k $ \kd -> kd { keySize = Nothing } 
+		| otherwise = k
 	afile = AssociatedFile (Just file)
diff --git a/doc/forum/Dropping_checksum_from_URL_key/comment_1_ce952a9db35d398a9e67adcd7f927d59._comment b/doc/forum/Dropping_checksum_from_URL_key/comment_1_ce952a9db35d398a9e67adcd7f927d59._comment
new file mode 100644
index 0000000000..4e09933a18
--- /dev/null
+++ b/doc/forum/Dropping_checksum_from_URL_key/comment_1_ce952a9db35d398a9e67adcd7f927d59._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-12T16:10:01Z"
+ content="""
+Migrating to URL will not do anything since they already are url keys.
+
+This could be scripted using `git-annex examinekey` to 
+convert such a key into one without a size, and then using
+`git-annex rekey`, which lets the new key for a file be specified.
+
+However, that command is a low level plumbing command, and does not copy
+over the url list from the old to the new key as migrate does (nor other
+metadata). So you would also have to use `git-annex addurl file url`
+afterwards to add the url, and use `git-annex metadata` if you have
+metadata. Very unergonomic.
+"""]]
diff --git a/doc/forum/Dropping_checksum_from_URL_key/comment_2_bb458296d464c2025f96502338f34000._comment b/doc/forum/Dropping_checksum_from_URL_key/comment_2_bb458296d464c2025f96502338f34000._comment
new file mode 100644
index 0000000000..65576564f5
--- /dev/null
+++ b/doc/forum/Dropping_checksum_from_URL_key/comment_2_bb458296d464c2025f96502338f34000._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2021-11-12T16:57:15Z"
+ content="""
+Implemented: `git-annex migrate --remove-size --backend=URL`
+
+Be sure to only run it on files using url keys, since it will also 
+remove sizes from other keys. (Or use `--inbackend=URL` with it.)
+
+Do note that `git-annex migrate` can only migrate files whose content
+is present. If you have never downloaded those urls, and `git-annex get`
+cannot download them now, because their size has changed, you
+won't be able to migrate data you don't have. In this case, re-running
+`git-annex addurl` with `--relaxed` seems like the only option.
+"""]]
diff --git a/doc/git-annex-migrate.mdwn b/doc/git-annex-migrate.mdwn
index 79b3434807..b05991930d 100644
--- a/doc/git-annex-migrate.mdwn
+++ b/doc/git-annex-migrate.mdwn
@@ -39,6 +39,18 @@ it's best to run migrate in all of them.
 
 * Also the [[git-annex-common-options]](1) can be used.
 
+* `--remove-size`
+
+  Keys often include the size of their content, which is generally a useful
+  thing. In fact, this command defaults to adding missing size information
+  to keys. With this option, the size information is removed instead.
+
+  One use of this option is to convert URL keys that were added
+  by `git-annex addurl --fast` to ones that would have been added if
+  that command was run with the `--relaxed` option. Eg:
+
+  	git-annex migrate --remove-size --backend=URL somefile
+
 # SEE ALSO
 
 [[git-annex]](1)
diff --git a/doc/git-annex-rekey.mdwn b/doc/git-annex-rekey.mdwn
index e3171f7000..85458bdc55 100644
--- a/doc/git-annex-rekey.mdwn
+++ b/doc/git-annex-rekey.mdwn
@@ -13,6 +13,9 @@ both the file, and the new key to use for it.
 
 Multiple pairs of file and key can be given in a single command line.
 
+Note that, unlike `git-annex migrate`, this does not copy over metadata,
+urls, and other such information from the old to the new key
+
 # OPTIONS
 
 * `--force`
@@ -37,6 +40,8 @@ Multiple pairs of file and key can be given in a single command line.
 
 [[git-annex]](1)
 
+[[git-annex-migrate]](1)
+
 # AUTHOR
 
 Joey Hess <id@joeyh.name>

diff --git a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_3_1dcf8d6a37bec46a7e7c2d9a65becf28._comment b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_3_1dcf8d6a37bec46a7e7c2d9a65becf28._comment
new file mode 100644
index 0000000000..ef52795d03
--- /dev/null
+++ b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_3_1dcf8d6a37bec46a7e7c2d9a65becf28._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="iimog@2c22a44141070c04b943932b697818a686859677"
+ nickname="iimog"
+ avatar="http://cdn.libravatar.org/avatar/e6b150bcd25f22fa74abde05e5d6d4eb"
+ subject="It works"
+ date="2021-11-12T08:14:32Z"
+ content="""
+Thank you so much for this. I can confirm that using the latest version, I can push to GitLab LFS with git-annex and it also works when used through DataLad. This opens a whole range of options for me, so thanks a million!
+"""]]

diff --git a/CHANGELOG b/CHANGELOG
index 4a0c58b277..6474b20204 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -11,6 +11,8 @@ git-annex (8.20211029) UNRELEASED; urgency=medium
   * uninit: Avoid error message when no commits have been made to the
     repository yet.
   * uninit: Avoid error message when there is no git-annex branch.
+  * git-lfs: Fix interoperability with gitlab's implementation of the
+    git-lfs protocol, which requests Content-Encoding chunked.
 
  -- Joey Hess <id@joeyh.name>  Mon, 01 Nov 2021 13:19:46 -0400
 
diff --git a/Remote/GitLFS.hs b/Remote/GitLFS.hs
index 941bf68ce8..29c745e155 100644
--- a/Remote/GitLFS.hs
+++ b/Remote/GitLFS.hs
@@ -457,7 +457,11 @@ store rs h = fileStorer $ \k src p -> getLFSEndpoint LFS.RequestUpload h >>= \ca
 		(req, sha256, size) <- mkUploadRequest rs k src
 		sendTransferRequest req endpoint >>= \case
 			Right resp -> do
-				body <- liftIO $ httpBodyStorer src p
+				let body (LFS.ServerSupportsChunks ssc) =
+					if ssc
+						then httpBodyStorerChunked src p
+						else RequestBodyIO $
+							httpBodyStorer src p
 				forM_ (LFS.objects resp) $
 					send body sha256 size
 			Left err -> giveup err
diff --git a/Remote/Helper/Http.hs b/Remote/Helper/Http.hs
index 3dc6598e5f..09e246b31f 100644
--- a/Remote/Helper/Http.hs
+++ b/Remote/Helper/Http.hs
@@ -37,6 +37,12 @@ httpBodyStorer src m = do
 	let streamer sink = withMeteredFile src m $ \b -> byteStringPopper b sink
 	return $ RequestBodyStream (fromInteger size) streamer
 
+-- Like httpBodyStorer, but generates a chunked request body.
+httpBodyStorerChunked :: FilePath -> MeterUpdate -> RequestBody
+httpBodyStorerChunked src m =
+	let streamer sink = withMeteredFile src m $ \b -> byteStringPopper b sink
+	in RequestBodyStreamChunked streamer
+
 byteStringPopper :: L.ByteString -> NeedsPopper () -> IO ()
 byteStringPopper b sink = do
 	mvar <- newMVar $ L.toChunks b
diff --git a/Utility/GitLFS.hs b/Utility/GitLFS.hs
index 831423502a..3df8ec953b 100644
--- a/Utility/GitLFS.hs
+++ b/Utility/GitLFS.hs
@@ -45,6 +45,7 @@ module Utility.GitLFS (
 	-- * Making transfers
 	downloadOperationRequest,
 	uploadOperationRequests,
+	ServerSupportsChunks(..),
 
 	-- * Endpoint discovery
 	Endpoint,
@@ -402,10 +403,10 @@ parseTransferResponse resp = case eitherDecode resp of
 
 -- | Builds a http request to perform a download.
 downloadOperationRequest :: DownloadOperation -> Maybe Request
-downloadOperationRequest = operationParamsRequest . download
+downloadOperationRequest = fmap fst . operationParamsRequest . download
 
 -- | Builds http request to perform an upload. The content to upload is
--- provided in the RequestBody, along with its SHA256 and size.
+-- provided, along with its SHA256 and size.
 --
 -- When the LFS server requested verification, there will be a second
 -- Request that does that; it should be run only after the upload has
@@ -413,8 +414,8 @@ downloadOperationRequest = operationParamsRequest . download
 --
 -- When the LFS server already contains the object, an empty list may be
 -- returned.
-uploadOperationRequests :: UploadOperation -> RequestBody -> SHA256 -> Integer -> Maybe [Request]
-uploadOperationRequests op content oid size = 
+uploadOperationRequests :: UploadOperation -> (ServerSupportsChunks -> RequestBody) -> SHA256 -> Integer -> Maybe [Request]
+uploadOperationRequests op mkcontent oid size = 
 	case (mkdlreq, mkverifyreq) of
 		(Nothing, _) -> Nothing
 		(Just dlreq, Nothing) -> Just [dlreq]
@@ -422,25 +423,40 @@ uploadOperationRequests op content oid size =
   where
 	mkdlreq = mkdlreq'
 		<$> operationParamsRequest (upload op)
-	mkdlreq' r = r
+	mkdlreq' (r, ssc) = r
 		{ method = "PUT"
-		, requestBody = content
+		, requestBody = mkcontent ssc
 		}
 	mkverifyreq = mkverifyreq'
 		<$> (operationParamsRequest =<< verify op)
-	mkverifyreq' r = addLfsJsonHeaders $ r
+	mkverifyreq' (r, _ssc) = addLfsJsonHeaders $ r
 		{ method = "POST"
 		, requestBody = RequestBodyLBS $ encode $
 			Verification oid size
 		}
 
-operationParamsRequest :: OperationParams -> Maybe Request
+-- | When the LFS server indicates that it supports Transfer-Encoding chunked,
+-- this will contain a true value, and the RequestBody provided to
+-- uploadOperationRequests may be created using RequestBodyStreamChunked.
+-- Otherwise, that should be avoided as the server may not support the
+-- chunked encoding.
+newtype ServerSupportsChunks = ServerSupportsChunks Bool
+
+operationParamsRequest :: OperationParams -> Maybe (Request, ServerSupportsChunks)
 operationParamsRequest ps = do
 	r <- parseRequest (T.unpack (href ps))
 	let headers = map convheader $ maybe [] M.toList (header ps)
-	return $ r { requestHeaders = headers }
+	let headers' = filter allowedheader headers
+	let ssc = ServerSupportsChunks $
+		any (== ("Transfer-Encoding", "chunked")) headers
+	return (r { requestHeaders = headers' }, ssc)
   where
 	convheader (k, v) = (CI.mk (E.encodeUtf8 k), E.encodeUtf8 v)
+	-- requestHeaders is not allowed to set Transfer-Encoding or 
+	-- Content-Length; copying those over blindly could request in a
+	-- malformed request.
+	allowedheader (k, _) = k /= "Transfer-Encoding"
+		&& k /= "Content-Length"
 
 type Url = T.Text
 
diff --git a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
index 968330f204..fdb2ada535 100644
--- a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
+++ b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
@@ -115,3 +115,5 @@ copy: 1 failed
 Yes, I'm using DataLad for some of my projects and I'm really impressed how it makes use of git-annex to solve many of the tasks that I struggled with pure git before.
 
 [[!tag projects/datalad]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_1_e821d8666ace3a760a2a335220844b29._comment b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_1_e821d8666ace3a760a2a335220844b29._comment
new file mode 100644
index 0000000000..a61d560585
--- /dev/null
+++ b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working/comment_1_e821d8666ace3a760a2a335220844b29._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2021-11-09T20:09:01Z"
+ content="""
+Let's see.. git-lfs endpoint discovery over ssh works. 
+
+The request to start a transfer works:
+
+	  host                 = "gitlab.com"
+	  path                 = "/joeyh/test.git/info/lfs/objects/batch"
+	...
+	[2021-11-10 12:06:35.409182815] (Remote.GitLFS) Status {statusCode = 200, statusMessage = "OK"}
+
+So it's the actual PUT that fails:
+
+	requestHeaders       = [("Authorization","<REDACTED>"),("Content-Type","application/octet-stream"),("Transfer-Encoding","chunked"),("User-Agent","git-annex/8.20211029-ga5a7d8433")]
+	path                 = "/joeyh/test.git/gitlab-lfs/objects/922d58c647a679e17ee7c30f7de0111b56b90e84129fa3663568b81822a2628a/30"
+
+Seems that the Transfer-Encoding chunked header is the problem.
+That header is provided by the git-lfs endpoint as one to include in
+the PUT (along with the Authorization header), and git-annex dutifully does
+include it. But it seems that does not make the PUT use that
+transfer encoding. And then in the server error, we see 
+"invalid chunked body".
+
+I tried filtering out the Transfer-Encoding header, and that does 
+fix the problem. But I dunno if that's the best fix. Should git-annex support
+Transfer-Encoding chunked?
+
+git-lfs has itself supported Transfer-Encoding chunked since 2015,
+see <https://github.com/git-lfs/git-lfs/issues/385>. That says
+"client may send data via chunked Transfer-Encoding when the server
+explicitly advertises that it's supported". Which is an interesting
+wording -- "may", "supported" -- implying it's not required to use it.
+
+The API docs <https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md>
+says the http headers are "Optional hash of String HTTP header key/value
+pairs to apply to the request". I think it means optional as in the
+server may optionally not send any, not necessarily 
+that applying them to the request is optional, but that's not really clear.
+(Surely a header like Authorization is not optional to include.)
+
+If the headers are not optional to include then the API would let the server
+specify any http headers at all, and the client has to send a PUT that includes
+those headers and that complies with them. So Transfer-Encoding deflate would
+need to use that compression method, etc.
+
+But looking at the git-lfs implementation, it only actually handles
+Transfer-Encoding chunked and not other values. I think it may also
+not include other headers than Authorization in the PUT?
+
+It seems possible there are other headers that might cause problems if they are
+blindly copied into the PUT. Content-Encoding is the only other obvious one,
+but who knows what may lurk in some odd corner of a HTTP spec.

(Diff truncated)

diff --git a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
index 96ef9e04c9..968330f204 100644
--- a/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
+++ b/doc/bugs/Git_LFS_as_a_special_remote_on_GitLab_not_working.mdwn
@@ -114,3 +114,4 @@ copy: 1 failed
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 Yes, I'm using DataLad for some of my projects and I'm really impressed how it makes use of git-annex to solve many of the tasks that I struggled with pure git before.
 
+[[!tag projects/datalad]]