Some ES downloads get stuck: citations.txt does not exist

[muttcg]

Some Elasticsearch downloads get stuck, logs show identical error:

>>> Invoking Main class now >>>

INFO: loading log4j config file log4j.properties.
INFO: log4j config file log4j.properties loaded successfully.
Setting [tez.application.tags] tag: oozie-ebea692ea0f668d527ec0dcf761171d
Setting [spark.yarn.tags] tag: oozie-ebea692ea0f668d527ec0dcf761171d
Fetching child yarn jobs
tag id : oozie-ebea692ea0f668d527ec0dcf761171d
Child yarn jobs are found - 
Main class        : org.gbif.occurrence.download.oozie.FromSearchDownloadAction
Arguments         :
             DWCA
             {
  "bool" : {
    "filter" : [
      {
        "bool" : {
          "filter" : [
            {
              "term" : {
                "gbifClassification.taxonKey" : {
                  "value" : "1000003",
                  "boost" : 1.0
                }
              }
            }
          ],
          "adjust_pure_negative" : true,
          "boost" : 1.0
        }
      },
      {
        "bool" : {
          "filter" : [
            {
              "term" : {
                "hasCoordinate" : {
                  "value" : "true",
                  "boost" : 1.0
                }
              }
            }
          ],
          "adjust_pure_negative" : true,
          "boost" : 1.0
        }
      },
      {
        "bool" : {
          "filter" : [
            {
              "term" : {
                "hasGeospatialIssue" : {
                  "value" : "false",
                  "boost" : 1.0
                }
              }
            }
          ],
          "adjust_pure_negative" : true,
          "boost" : 1.0
        }
      },
      {
        "bool" : {
          "filter" : [
            {
              "term" : {
                "occurrenceStatus" : {
                  "value" : "PRESENT",
                  "boost" : 1.0
                }
              }
            }
          ],
          "adjust_pure_negative" : true,
          "boost" : 1.0
        }
      }
    ],
    "adjust_pure_negative" : true,
    "boost" : 1.0
  }
}
             0266719-220831081235567
             {"type":"and","predicates":[{"type":"equals","key":"TAXON_KEY","value":"1000003","matchCase":false},{"type":"equals","key":"HAS_COORDINATE","value":"true","matchCase":false},{"type":"equals","key":"HAS_GEOSPATIAL_ISSUE","value":"false","matchCase":false},{"type":"equals","key":"OCCURRENCE_STATUS","value":"PRESENT","matchCase":false}]}
             0266719_220831081235567
             lyrodus06
             Occurrence
             null

[ERROR] [01/31/2023 03:48:13.333] [DownloadSystem0266719-220831081235567-akka.actor.default-dispatcher-6] [akka://DownloadSystem0266719-220831081235567/user/DownloadMaster0266719-220831081235567] java.io.FileNotFoundException: File tmp/0266719-220831081235567/0266719_220831081235567_citations.txt does not exist
java.lang.RuntimeException: java.io.FileNotFoundException: File tmp/0266719-220831081235567/0266719_220831081235567_citations.txt does not exist
	at org.gbif.occurrence.download.file.dwca.archive.DownloadArchiveBuilder.buildArchive(DownloadArchiveBuilder.java:128)
	at org.gbif.occurrence.download.file.dwca.DwcaArchiveBuilder.buildArchive(DwcaArchiveBuilder.java:203)
	at org.gbif.occurrence.download.file.dwca.akka.DwcaDownloadAggregator.aggregate(DwcaDownloadAggregator.java:102)
	at org.gbif.occurrence.download.file.DownloadMaster.aggregateAndShutdown(DownloadMaster.java:125)
	at org.gbif.occurrence.download.file.DownloadMaster.runActors(DownloadMaster.java:211)
	at org.gbif.occurrence.download.file.DownloadMaster.onReceive(DownloadMaster.java:154)
	at akka.actor.UntypedActor$$anonfun$receive$1.apply(UntypedActor.scala:154)
	at akka.actor.UntypedActor$$anonfun$receive$1.apply(UntypedActor.scala:153)
	at akka.actor.Actor$class.apply(Actor.scala:318)
	at akka.actor.UntypedActor.apply(UntypedActor.scala:93)
	at akka.actor.ActorCell.invoke(ActorCell.scala:626)
	at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:197)
	at akka.dispatch.Mailbox.run(Mailbox.scala:179)
	at akka.dispatch.ForkJoinExecutorConfigurator$MailboxExecutionTask.exec(AbstractDispatcher.scala:516)
	at akka.jsr166y.ForkJoinTask.doExec(ForkJoinTask.java:259)
	at akka.jsr166y.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:975)
	at akka.jsr166y.ForkJoinPool.runWorker(ForkJoinPool.java:1479)
	at akka.jsr166y.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:104)
Caused by: java.io.FileNotFoundException: File tmp/0266719-220831081235567/0266719_220831081235567_citations.txt does not exist
	at org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:414)
	at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1566)
	at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1609)
	at org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:667)
	at org.gbif.occurrence.download.file.dwca.archive.CitationFileReader.read(CitationFileReader.java:66)
	at org.gbif.occurrence.download.file.dwca.archive.DownloadArchiveBuilder.buildArchive(DownloadArchiveBuilder.java:119)
	... 17 more

Heart beat
Heart beat
Heart beat
Heart beat
Heart beat
Heart beat