diff --git a/reports/benchmarks/2024-01-29_12-07-12/README.md b/reports/benchmarks/2024-01-29_12-07-12/README.md new file mode 100644 index 000000000..9d434bb71 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/README.md @@ -0,0 +1,18 @@ +# RAGStack - Benchmarks Report - 2024-01-29 12:07:09 + +| Test Case | p50 | p90 | p95 | p99 | +|---|---|---|---|---| +| embeddings_batch100_chunk256-openai_ada002 | 3.06 | 3.06 | 3.06 | 3.06 | +| embeddings_batch100_chunk512-openai_ada002 | 3.2 | 3.2 | 3.2 | 3.2 | +| embeddings_batch50_chunk256-openai_ada002 | 3.55 | 3.55 | 3.55 | 3.55 | +| embeddings_batch50_chunk512-openai_ada002 | 3.55 | 3.55 | 3.55 | 3.55 | +| embeddings_batch10_chunk256-openai_ada002 | 5.4 | 5.4 | 5.4 | 5.4 | +| embeddings_batch50_chunk512-nvidia_nvolveqa40k | 5.52 | 5.52 | 5.52 | 5.52 | +| embeddings_batch100_chunk256-nvidia_nvolveqa40k | 6.2 | 6.2 | 6.2 | 6.2 | +| embeddings_batch100_chunk512-nvidia_nvolveqa40k | 6.21 | 6.21 | 6.21 | 6.21 | +| embeddings_batch10_chunk512-openai_ada002 | 6.92 | 6.92 | 6.92 | 6.92 | +| embeddings_batch50_chunk256-nvidia_nvolveqa40k | 7.2 | 7.2 | 7.2 | 7.2 | +| embeddings_batch10_chunk512-nvidia_nvolveqa40k | 9.1 | 9.1 | 9.1 | 9.1 | +| embeddings_batch10_chunk256-nvidia_nvolveqa40k | 9.12 | 9.12 | 9.12 | 9.12 | + + \ No newline at end of file diff --git a/reports/benchmarks/2024-01-29_12-07-12/benchmarks.log b/reports/benchmarks/2024-01-29_12-07-12/benchmarks.log new file mode 100644 index 000000000..b783f490c --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/benchmarks.log @@ -0,0 +1,76 @@ +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings "HTTP/1.1 200 OK" +INFO:root:Starting test case +INFO:root:Starting test case diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..1875cd7db --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[6.202193088000001],"warmups":[[1,8.033775862000027]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch100_chunk256 nvidia_nvolveqa40k","command_max_rss":83632128,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=2935 MHz; 1=2877 MHz; 2=3242 MHz; 3=3297 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:47.762126","duration":14.306412134000027,"hostname":"fv-az525-874","load_avg_1min":0.48,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":307.76432180404663},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-openai_ada002.json new file mode 100644 index 000000000..707bdd21d --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk256-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[3.0632637260000024],"warmups":[[1,3.056866897999953]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch100_chunk256 openai_ada002","command_max_rss":151453696,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3243 MHz; 1=3199 MHz; 2=3332 MHz; 3=3019 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:33.327916","duration":6.1933870070000125,"hostname":"fv-az525-874","load_avg_1min":0.62,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":293.33025908470154},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..17e4a7c2e --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[6.214858456000002],"warmups":[[1,6.478936356999952]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch100_chunk512 nvidia_nvolveqa40k","command_max_rss":83763200,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3244 MHz; 1=3255 MHz; 2=2814 MHz; 3=2966 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:07:07.330620","duration":12.766234963999977,"hostname":"fv-az525-874","load_avg_1min":0.7,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":327.3327782154083},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-openai_ada002.json new file mode 100644 index 000000000..4a56c8dda --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch100_chunk512-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[3.2019995630000153],"warmups":[[1,3.2139059389999716]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch100_chunk512 openai_ada002","command_max_rss":153448448,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3242 MHz; 1=3213 MHz; 2=2789 MHz; 3=2698 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:54.438789","duration":6.488751931999957,"hostname":"fv-az525-874","load_avg_1min":0.52,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":314.44088101387024},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..cf1c118b4 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[9.117978915999998],"warmups":[[1,9.415244596999997]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch10_chunk256 nvidia_nvolveqa40k","command_max_rss":81498112,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=2880 MHz; 1=3058 MHz; 2=2889 MHz; 3=3059 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:05:18.664709","duration":18.610249519000007,"hostname":"fv-az525-874","load_avg_1min":0.56,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":218.66720867156982},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-openai_ada002.json new file mode 100644 index 000000000..2f9f09c28 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk256-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[5.398796991000012],"warmups":[[1,7.291147414999983]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch10_chunk256 openai_ada002","command_max_rss":152076288,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=2868 MHz; 1=2929 MHz; 2=3242 MHz; 3=3333 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:04:59.923632","duration":12.76397141199999,"hostname":"fv-az525-874","load_avg_1min":0.67,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":199.92592096328735},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..d6236cc52 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[9.098559965000021],"warmups":[[1,7.789102850000006]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch10_chunk512 nvidia_nvolveqa40k","command_max_rss":81600512,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3078 MHz; 1=3311 MHz; 2=3236 MHz; 3=2893 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:05:48.432511","duration":16.958478074999988,"hostname":"fv-az525-874","load_avg_1min":0.4,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":248.43471765518188},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-openai_ada002.json new file mode 100644 index 000000000..3db50dad2 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch10_chunk512-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[6.916698567999987],"warmups":[[1,5.499823508999981]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch10_chunk512 openai_ada002","command_max_rss":149278720,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3251 MHz; 1=3244 MHz; 2=2972 MHz; 3=2653 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:05:31.347007","duration":12.489086231000016,"hostname":"fv-az525-874","load_avg_1min":0.51,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":231.34922337532043},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..dd95bd5d3 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[7.2019749669999555],"warmups":[[1,5.734251422999989]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch50_chunk256 nvidia_nvolveqa40k","command_max_rss":83742720,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=2888 MHz; 1=3234 MHz; 2=3238 MHz; 3=2878 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:08.749461","duration":13.007608946000005,"hostname":"fv-az525-874","load_avg_1min":0.35,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":268.7517671585083},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-openai_ada002.json new file mode 100644 index 000000000..e8ac50b01 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk256-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[3.547764126000004],"warmups":[[1,3.3797298540000043]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch50_chunk256 openai_ada002","command_max_rss":149700608,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3239 MHz; 1=2971 MHz; 2=2606 MHz; 3=2960 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:05:55.617879","duration":6.999468656000005,"hostname":"fv-az525-874","load_avg_1min":0.37,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":255.6200726032257},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-nvidia_nvolveqa40k.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-nvidia_nvolveqa40k.json new file mode 100644 index 000000000..e18bf97fb --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-nvidia_nvolveqa40k.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[5.517936516999953],"warmups":[[1,5.370841561999953]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch50_chunk512 nvidia_nvolveqa40k","command_max_rss":83763200,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=3273 MHz; 1=3243 MHz; 2=2908 MHz; 3=3210 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:26.939454","duration":10.95967594199999,"hostname":"fv-az525-874","load_avg_1min":0.59,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":286.94176840782166},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-openai_ada002.json b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-openai_ada002.json new file mode 100644 index 000000000..07b8f6c69 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/embeddings_batch50_chunk512-openai_ada002.json @@ -0,0 +1 @@ +{"benchmarks":[{"runs":[{"values":[3.5479530439999962],"warmups":[[1,3.2966138630000046]]}]}],"metadata":{"aslr":"Full randomization","boot_time":"2024-01-29 12:01:40","command":"/home/runner/.cache/pypoetry/virtualenvs/ragstack-e2e-tests-KLvcJhlJ-py3.11/bin/python /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/testcases.py /home/runner/work/ragstack-ai/ragstack-ai/ragstack-e2e-tests/benchmarks/reports/benchmarks.log embeddings_batch50_chunk512 openai_ada002","command_max_rss":149684224,"cpu_config":"idle:none","cpu_count":4,"cpu_freq":"0=2690 MHz; 1=3236 MHz; 2=2912 MHz; 3=3237 MHz","cpu_model_name":"AMD EPYC 7763 64-Core Processor","date":"2024-01-29 12:06:15.853579","duration":6.916700273000004,"hostname":"fv-az525-874","load_avg_1min":0.4,"loops":1,"name":"command","perf_version":"2.6.2","platform":"Linux-6.2.0-1018-azure-x86_64-with-glibc2.35","runnable_threads":1,"unit":"second","uptime":275.8559684753418},"version":"1.0"} diff --git a/reports/benchmarks/2024-01-29_12-07-12/index.html b/reports/benchmarks/2024-01-29_12-07-12/index.html new file mode 100644 index 000000000..88703cb1e --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/index.html @@ -0,0 +1,24 @@ + + + + + + + + +
+

RAGStack - Benchmarks Report - 2024-01-29 12:07:09

+
+
+
+
+

Table

+
Test Casep50p90p95p99
embeddings_batch100_chunk256-openai_ada0023.063.063.063.06
embeddings_batch100_chunk512-openai_ada0023.23.23.23.2
embeddings_batch50_chunk256-openai_ada0023.553.553.553.55
embeddings_batch50_chunk512-openai_ada0023.553.553.553.55
embeddings_batch10_chunk256-openai_ada0025.45.45.45.4
embeddings_batch50_chunk512-nvidia_nvolveqa40k5.525.525.525.52
embeddings_batch100_chunk256-nvidia_nvolveqa40k6.26.26.26.2
embeddings_batch100_chunk512-nvidia_nvolveqa40k6.216.216.216.21
embeddings_batch10_chunk512-openai_ada0026.926.926.926.92
embeddings_batch50_chunk256-nvidia_nvolveqa40k7.27.27.27.2
embeddings_batch10_chunk512-nvidia_nvolveqa40k9.19.19.19.1
embeddings_batch10_chunk256-nvidia_nvolveqa40k9.129.129.129.12
+
+
+

Plot

+ +
+
+
+ \ No newline at end of file diff --git a/reports/benchmarks/2024-01-29_12-07-12/plot.svg b/reports/benchmarks/2024-01-29_12-07-12/plot.svg new file mode 100644 index 000000000..767c85d02 --- /dev/null +++ b/reports/benchmarks/2024-01-29_12-07-12/plot.svg @@ -0,0 +1,1860 @@ + + + + + + + + 2024-01-29T12:07:09.449929 + image/svg+xml + + + Matplotlib v3.8.2, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +