-
-
Notifications
You must be signed in to change notification settings - Fork 337
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Cleanup Testing & Retry logic (#3477)
* Consolidate all Scala-level tries into one `mill.api.Retry` helper, add some minimal unit tests, and use it throughout Mill * Swap out `lambdatest` for `junit` so we can use the standard JUnit retry facilities on `FileToStreamTailerTest` (nothing built in, but googling brings up lots of examples to copy-paste) * Remove `os.temp.dir` from Integration tester `workspacePath` and replace it with a shorter `run-$i` folder that serves the same purpose * Try to be more aggressive at cleaning up prior `out/` folder to force the processes to terminate
- Loading branch information
Showing
22 changed files
with
430 additions
and
266 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package mill.api | ||
|
|
||
| import java.util.concurrent.TimeUnit | ||
| import scala.concurrent.duration.Duration | ||
| import scala.concurrent.{Await, Promise} | ||
|
|
||
| case class Retry( | ||
| count: Int = 5, | ||
| backoffMillis: Long = 10, | ||
| backoffMultiplier: Double = 2.0, | ||
| timeoutMillis: Long = -1, | ||
| filter: (Int, Throwable) => Boolean = (_, _) => true | ||
| ) { | ||
|
|
||
| /** | ||
| * Generic retry functionality | ||
| * | ||
| * @param count How many times to retry before giving up | ||
| * @param backoffMillis What is the initial backoff time | ||
| * @param backoffMultiplier How much to multiply the initial backoff each time | ||
| * @param timeoutMillis How much time we want to allow [[t]] to run. If passed, | ||
| * runs [[t]] in a separate thread and throws a `TimeoutException` | ||
| * if it takes too long | ||
| * @param filter Whether or not we want to retry a given exception at a given retryCount; | ||
| * defaults to `true` to retry all exceptions, but can be made more fine-grained | ||
| * to only retry specific exceptions, or log them together with the retryCount | ||
| * @param t The code block that we want to retry | ||
| * @return the value of evaluating [[t]], or throws an exception if evaluating | ||
| * [[t]] fails more than [[count]] times | ||
| */ | ||
| def apply[T](t: => T): T = { | ||
| indexed(i => t) | ||
| } | ||
|
|
||
| def indexed[T](t: Int => T): T = { | ||
| def rec(retryCount: Int, currentBackoffMillis: Long): T = { | ||
| try { | ||
| if (timeoutMillis == -1) t(retryCount) | ||
| else { | ||
| val result = Promise[T] | ||
| val thread = new Thread(() => { | ||
| result.complete(scala.util.Try(t(retryCount))) | ||
| }) | ||
| thread.start() | ||
| Await.result(result.future, Duration.apply(timeoutMillis, TimeUnit.MILLISECONDS)) | ||
| } | ||
| } catch { | ||
| case e: Throwable if retryCount < count && filter(retryCount + 1, e) => | ||
| Thread.sleep(currentBackoffMillis) | ||
| rec(retryCount + 1, (currentBackoffMillis * backoffMultiplier).toInt) | ||
| } | ||
| } | ||
|
|
||
| rec(0, backoffMillis) | ||
| } | ||
| } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package mill.api | ||
|
|
||
| import utest._ | ||
|
|
||
| object RetryTests extends TestSuite { | ||
| val tests: Tests = Tests { | ||
| test("fail") { | ||
| var count = 0 | ||
| try { | ||
| Retry() { | ||
| count += 1 | ||
| throw new Exception("boom") | ||
| } | ||
| } catch { | ||
| case ex => | ||
| assert(ex.getMessage == "boom") | ||
| } | ||
|
|
||
| assert(count == 6) // 1 original try + 5 retries | ||
| } | ||
| test("succeed") { | ||
| var count = 0 | ||
| Retry() { | ||
| count += 1 | ||
| if (count < 3) throw new Exception("boom") | ||
| } | ||
| assert(count == 3) | ||
| } | ||
| test("filter") { | ||
| var count = 0 | ||
| try { | ||
| Retry( | ||
| filter = { | ||
| case (i, ex: RuntimeException) => true | ||
| case _ => false | ||
| } | ||
| ) { | ||
| count += 1 | ||
| if (count < 3) throw new RuntimeException("boom") | ||
| else throw new Exception("foo") | ||
| } | ||
| } catch { | ||
| case e: Exception => | ||
| assert(e.getMessage == "foo") | ||
| } | ||
| assert(count == 3) | ||
| } | ||
| test("timeout") { | ||
| test("fail") { | ||
| var count = 0 | ||
| try { | ||
| Retry(timeoutMillis = 100) { | ||
| count += 1 | ||
| Thread.sleep(1000) | ||
| } | ||
| } catch { | ||
| case e: Exception => | ||
| assert(e.getMessage == "Future timed out after [100 milliseconds]") | ||
| } | ||
|
|
||
| assert(count == 6) // 1 original try + 5 retries | ||
| } | ||
| test("success") { | ||
| var count = 0 | ||
| Retry(timeoutMillis = 100) { | ||
| count += 1 | ||
| if (count < 3) Thread.sleep(1000) | ||
| } | ||
|
|
||
| assert(count == 3) | ||
| } | ||
|
|
||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.