diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 0000000..12c93b9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,67 @@ +name: "\U0001F41B Bug Report" +description: Submit a bug report to help us improve text-generation-inference +body: + - type: textarea + id: system-info + attributes: + label: System Info + description: | + Please share your system info with us (`text-generation-launcher --env` if installed locally). + The full command line used that causes issues: + OS version: + Rust version (if self-compiling, `cargo version`): + Model being used (`curl 127.0.0.1:8080/info | jq`): + If local model please explicit the kind of model and/or equivalents. + Hardware used (GPUs, how many, on which cloud) (`nvidia-smi`): + Deployment specificities (Kubernetes, EKS, AKS, any particular deployments): + The current version being used: + + placeholder: text-generation-inference version, platform, python version, ... + validations: + required: true + + - type: checkboxes + id: information-scripts-examples + attributes: + label: Information + description: 'The problem arises when using:' + options: + - label: "Docker" + - label: "The CLI directly" + + - type: checkboxes + id: information-tasks + attributes: + label: Tasks + description: "The thing I am working on is:" + options: + - label: "An officially supported command" + - label: "My own modifications" + + - type: textarea + id: reproduction + validations: + required: true + attributes: + label: Reproduction + description: | + Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet. + If you have code snippets, error messages, stack traces please provide them here as well. + Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting + Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code. + + placeholder: | + Steps to reproduce the behavior: + + 1. + 2. + 3. + + + - type: textarea + id: expected-behavior + validations: + required: true + attributes: + label: Expected behavior + description: "A clear and concise description of what you would expect to happen." diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..e647772 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,2 @@ +blank_issues_enabled: true +version: 2.1 diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..5abc156 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,31 @@ +name: "\U0001F680 Feature request" +description: Submit a proposal/request for a new text-generation-inference feature +labels: [ "feature" ] +body: + - type: textarea + id: feature-request + validations: + required: true + attributes: + label: Feature request + description: | + A clear and concise description of the feature proposal. Please provide a link to the paper and code in case they exist. + + - type: textarea + id: motivation + validations: + required: true + attributes: + label: Motivation + description: | + Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too. + + + - type: textarea + id: contribution + validations: + required: true + attributes: + label: Your contribution + description: | + Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/text-generation-inference/blob/main/CONTRIBUTING.md) diff --git a/.github/ISSUE_TEMPLATE/new-model-addition.yml b/.github/ISSUE_TEMPLATE/new-model-addition.yml new file mode 100644 index 0000000..2f3476d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-model-addition.yml @@ -0,0 +1,31 @@ +name: "\U0001F31F New model addition" +description: Submit a proposal/request to implement a new model +labels: [ "New model" ] + +body: + - type: textarea + id: description-request + validations: + required: true + attributes: + label: Model description + description: | + Put any and all important information relative to the model + + - type: checkboxes + id: information-tasks + attributes: + label: Open source status + description: | + Please note that if the model implementation isn't available or if the weights aren't open-source, we are less likely to implement it in `transformers`. + options: + - label: "The model implementation is available" + - label: "The model weights are available" + + - type: textarea + id: additional-info + attributes: + label: Provide useful links for the implementation + description: | + Please provide information regarding the implementation, the weights, and the authors. + Please mention the authors by @gh-username if you're aware of their usernames. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..ad5b98a --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,40 @@ +# What does this PR do? + + + + + +Fixes # (issue) + + +## Before submitting +- [ ] This PR fixes a typo or improves the docs (you can dismiss the other checks if that's the case). +- [ ] Did you read the [contributor guideline](https://github.com/huggingface/transformers/blob/main/CONTRIBUTING.md#start-contributing-pull-requests), + Pull Request section? +- [ ] Was this discussed/approved via a Github issue or the [forum](https://discuss.huggingface.co/)? Please add a link + to it if that's the case. +- [ ] Did you make sure to update the documentation with your changes? Here are the + [documentation guidelines](https://github.com/huggingface/transformers/tree/main/docs), and + [here are tips on formatting docstrings](https://github.com/huggingface/transformers/tree/main/docs#writing-source-documentation). +- [ ] Did you write any new necessary tests? + + +## Who can review? + +Anyone in the community is free to review the PR once the tests have passed. Feel free to tag +members/contributors who may be interested in your PR. + + diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 67d1c73..289e4f6 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -97,6 +97,7 @@ jobs: platforms: 'linux/amd64' build-args: | GIT_SHA=${{ env.GITHUB_SHA }} + DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/community/text-generation-inference:cache,mode=max diff --git a/Cargo.lock b/Cargo.lock index e240fe9..58142d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1428,6 +1428,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2063,6 +2072,15 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.37.11" @@ -2136,6 +2154,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + [[package]] name = "serde" version = "1.0.160" @@ -2345,6 +2369,20 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sysinfo" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2f3ca6693feb29a89724516f016488e9aafc7f37264f898593ee4b942f31b" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "winapi", +] + [[package]] name = "tar" version = "0.4.38" @@ -2399,6 +2437,7 @@ dependencies = [ "subprocess", "tracing", "tracing-subscriber", + "vergen", ] [[package]] @@ -2985,7 +3024,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1b86a8af1dedf089b1c78338678e4c7492b6045649042d94faf19690499d236" dependencies = [ "anyhow", + "rustc_version", "rustversion", + "sysinfo", "time", ] diff --git a/Dockerfile b/Dockerfile index c65d3de..d59acc1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG GIT_SHA +ARG DOCKER_LABEL RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \ curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \ diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 5944a36..885dcd9 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -17,3 +17,6 @@ tracing-subscriber = { version = "0.3.16", features = ["json"] } [dev-dependencies] float_eq = "1.0.1" reqwest = { version = "0.11.14", features = ["blocking", "json"] } + +[build-dependencies] +vergen = { version = "8.0.0", features = ["build", "cargo", "git", "gitcl", "rustc", "si"] } diff --git a/launcher/build.rs b/launcher/build.rs new file mode 100644 index 0000000..71d2c0c --- /dev/null +++ b/launcher/build.rs @@ -0,0 +1,29 @@ +use std::error::Error; +use vergen::EmitBuilder; + +fn main() -> Result<(), Box> { + // Emit cargo and rustc compile time values + EmitBuilder::builder().all_cargo().all_rustc().emit()?; + + // Try to get the git sha from the local git repository + if EmitBuilder::builder() + .fail_on_error() + .git_sha(false) + .emit() + .is_err() + { + // Unable to get the git sha + if let Ok(sha) = std::env::var("GIT_SHA") { + // Set it from an env var + println!("cargo:rustc-env=VERGEN_GIT_SHA={sha}"); + } + } + + // Set docker label if present + if let Ok(label) = std::env::var("DOCKER_LABEL") { + // Set it from an env var + println!("cargo:rustc-env=DOCKER_LABEL={label}"); + } + + Ok(()) +} diff --git a/launcher/src/env_runtime.rs b/launcher/src/env_runtime.rs new file mode 100644 index 0000000..9dbc83f --- /dev/null +++ b/launcher/src/env_runtime.rs @@ -0,0 +1,45 @@ +use std::fmt; +use std::process::Command; + +pub(crate) struct Env { + cargo_target: &'static str, + cargo_version: &'static str, + git_sha: &'static str, + docker_label: &'static str, + nvidia_env: String, +} + +impl Env { + pub fn new() -> Self { + let nvidia_env = nvidia_smi(); + + Self { + nvidia_env: nvidia_env.unwrap_or("N/A".to_string()), + cargo_target: env!("VERGEN_CARGO_TARGET_TRIPLE"), + cargo_version: env!("VERGEN_RUSTC_SEMVER"), + git_sha: option_env!("VERGEN_GIT_SHA").unwrap_or("N/A"), + docker_label: option_env!("DOCKER_LABEL").unwrap_or("N/A"), + } + } +} + +impl fmt::Display for Env { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Runtime environment:")?; + + writeln!(f, "Target: {}", self.cargo_target)?; + writeln!(f, "Cargo version: {}", self.cargo_version)?; + writeln!(f, "Commit sha: {}", self.git_sha)?; + writeln!(f, "Docker label: {}", self.docker_label)?; + write!(f, "nvidia-smi:\n{}", self.nvidia_env)?; + + Ok(()) + } +} + +fn nvidia_smi() -> Option { + let output = Command::new("nvidia-smi").output().ok()?; + let nvidia_smi = String::from_utf8(output.stdout).ok()?; + let output = nvidia_smi.replace('\n', "\n "); + Some(output.trim().to_string()) +} diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 77b0cf8..736aa5a 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -14,6 +14,8 @@ use std::time::{Duration, Instant}; use std::{fs, io}; use subprocess::{ExitStatus, Popen, PopenConfig, PopenError, Redirection}; +mod env_runtime; + /// App Configuration #[derive(Parser, Debug)] #[clap(author, version, about, long_about = None)] @@ -200,6 +202,10 @@ struct Args { watermark_gamma: Option, #[clap(long, env)] watermark_delta: Option, + + /// Display a lot of information about your runtime environment + #[clap(long, short, action)] + env: bool, } #[derive(Debug)] @@ -829,6 +835,11 @@ fn main() -> Result<(), LauncherError> { tracing_subscriber::fmt().compact().init(); } + if args.env { + let env_runtime = env_runtime::Env::new(); + tracing::info!("{}", env_runtime); + } + tracing::info!("{:?}", args); let num_shard = find_num_shards(args.sharded, args.num_shard); diff --git a/router/build.rs b/router/build.rs index 1b1fdc8..f5eb8a2 100644 --- a/router/build.rs +++ b/router/build.rs @@ -15,5 +15,12 @@ fn main() -> Result<(), Box> { println!("cargo:rustc-env=VERGEN_GIT_SHA={sha}"); } } + + // Set docker label if present + if let Ok(label) = std::env::var("DOCKER_LABEL") { + // Set it from an env var + println!("cargo:rustc-env=DOCKER_LABEL={label}"); + } + Ok(()) } diff --git a/router/src/lib.rs b/router/src/lib.rs index c2ff669..080dc4f 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -57,6 +57,8 @@ pub struct Info { pub version: &'static str, #[schema(nullable = true, example = "null")] pub sha: Option<&'static str>, + #[schema(nullable = true, example = "null")] + pub docker_label: Option<&'static str>, } #[derive(Clone, Debug, Deserialize, ToSchema)] diff --git a/router/src/server.rs b/router/src/server.rs index f25fa2b..c5fc41e 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -635,6 +635,7 @@ pub async fn run( validation_workers, version: env!("CARGO_PKG_VERSION"), sha: option_env!("VERGEN_GIT_SHA"), + docker_label: option_env!("DOCKER_LABEL"), }; // Create router