Mirrors
/
hf_text-generation-inference
mirror of https://github.com/huggingface/text-generation-inference.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: Add variable results file location

This commit is contained in:
Hugo Larcher 2024-10-03 11:34:38 +02:00
parent d30266dc3c
commit 6ae04672b6
No known key found for this signature in database
GPG Key ID: 3DAF63124699CA2B
2 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/load_test.yaml vendored
View File

@ -44,7 +44,7 @@ jobs:
export PATH="$HOME/.local/bin:$PATH" export PATH="$HOME/.local/bin:$PATH"
cd load_tests cd load_tests
poetry install poetry install
poetry run python benchmarks.py poetry run python benchmarks.py --sha ${{ github.sha }} --results-file "s3://text-generation-inference-ci/benchmarks/ci/${{ github.sha }}.parquet"
shell: bash shell: bash
env: env:
HF_TOKEN: ${{ secrets.HF_TOKEN_BENCHMARK }} HF_TOKEN: ${{ secrets.HF_TOKEN_BENCHMARK }}

18
load_tests/benchmarks.py
View File

@ -1,3 +1,4 @@
import argparse
import datetime import datetime
import json import json
import os import os
@ -162,7 +163,7 @@ def build_df(model: str, data_files: dict[str, str]) -> pd.DataFrame:
return df return df
def main(): def main(sha, results_file):
results_dir = 'results' results_dir = 'results'
# get absolute path # get absolute path
results_dir = os.path.join(os.path.dirname(__file__), results_dir) results_dir = os.path.join(os.path.dirname(__file__), results_dir)
@ -172,7 +173,6 @@ def main():
# ('meta-llama/Llama-3.1-70B-Instruct', 4), # ('meta-llama/Llama-3.1-70B-Instruct', 4),
# ('mistralai/Mixtral-8x7B-Instruct-v0.1', 2), # ('mistralai/Mixtral-8x7B-Instruct-v0.1', 2),
] ]
sha = os.environ.get('GITHUB_SHA')
success = True success = True
for model in models: for model in models:
tgi_runner = TGIDockerRunner(model[0]) tgi_runner = TGIDockerRunner(model[0])
@ -225,8 +225,18 @@ def main():
df = pd.concat([df, build_df(directory.split('/')[-1], data_files)]) df = pd.concat([df, build_df(directory.split('/')[-1], data_files)])
df['device'] = get_gpu_name() df['device'] = get_gpu_name()
df['error_rate'] = df['failed_requests'] / (df['failed_requests'] + df['successful_requests']) * 100.0 df['error_rate'] = df['failed_requests'] / (df['failed_requests'] + df['successful_requests']) * 100.0
df.to_parquet(f's3://text-generation-inference-ci/benchmarks/ci/{sha}.parquet') df.to_parquet(results_file)
if __name__ == "__main__": if __name__ == "__main__":
main() parser = argparse.ArgumentParser()
parser.add_argument("--sha", help="SHA of the commit to add to the results", required=True)
parser.add_argument("--results-file",
help="The file where to store the results, can be a local file or a s3 path")
args = parser.parse_args()
if args.results_file is None:
results_file = f'{args.sha}.parquet'
else:
results_file = args.results_file
main(args.sha, results_file)