Add automatic doc sorting (#1940)
* automatically sort docs * add new check toc doc * add new check toc doc * add * add new check toc doc * add * add new check toc doc * correct * finalize
This commit is contained in:
Patrick von Platen
GitHub
parent
9f4c4f5e82
commit
aba2a65d6a
2
Makefile
2
Makefile
|
|
@ -45,12 +45,14 @@ quality:
|
|||
isort --check-only $(check_dirs)
|
||||
flake8 $(check_dirs)
|
||||
doc-builder style src/diffusers docs/source --max_len 119 --check_only --path_to_docs docs/source
|
||||
python utils/check_doc_toc.py
|
||||
|
||||
# Format source code automatically and check is there are any problems left that need manual fixing
|
||||
|
||||
extra_style_checks:
|
||||
python utils/custom_init_isort.py
|
||||
doc-builder style src/diffusers docs/source --max_len 119 --path_to_docs docs/source
|
||||
python utils/check_doc_toc.py --fix_and_overwrite
|
||||
|
||||
# this target runs checks on all files and potentially modifies some of them
|
||||
|
||||
|
|
|
|||
|
|
@ -1,195 +1,194 @@
|
|||
- sections:
|
||||
- local: index
|
||||
title: "🧨 Diffusers"
|
||||
title: 🧨 Diffusers
|
||||
- local: quicktour
|
||||
title: "Quicktour"
|
||||
title: Quicktour
|
||||
- local: stable_diffusion
|
||||
title: "Stable Diffusion"
|
||||
title: Stable Diffusion
|
||||
- local: installation
|
||||
title: "Installation"
|
||||
title: "Get started"
|
||||
title: Installation
|
||||
title: Get started
|
||||
- sections:
|
||||
- sections:
|
||||
- local: using-diffusers/loading
|
||||
title: "Loading Pipelines, Models, and Schedulers"
|
||||
title: Loading Pipelines, Models, and Schedulers
|
||||
- local: using-diffusers/schedulers
|
||||
title: "Using different Schedulers"
|
||||
title: Using different Schedulers
|
||||
- local: using-diffusers/configuration
|
||||
title: "Configuring Pipelines, Models, and Schedulers"
|
||||
title: Configuring Pipelines, Models, and Schedulers
|
||||
- local: using-diffusers/custom_pipeline_overview
|
||||
title: "Loading and Adding Custom Pipelines"
|
||||
title: "Loading & Hub"
|
||||
title: Loading and Adding Custom Pipelines
|
||||
title: Loading & Hub
|
||||
- sections:
|
||||
- local: using-diffusers/unconditional_image_generation
|
||||
title: "Unconditional Image Generation"
|
||||
title: Unconditional Image Generation
|
||||
- local: using-diffusers/conditional_image_generation
|
||||
title: "Text-to-Image Generation"
|
||||
title: Text-to-Image Generation
|
||||
- local: using-diffusers/img2img
|
||||
title: "Text-Guided Image-to-Image"
|
||||
title: Text-Guided Image-to-Image
|
||||
- local: using-diffusers/inpaint
|
||||
title: "Text-Guided Image-Inpainting"
|
||||
title: Text-Guided Image-Inpainting
|
||||
- local: using-diffusers/depth2img
|
||||
title: "Text-Guided Depth-to-Image"
|
||||
title: Text-Guided Depth-to-Image
|
||||
- local: using-diffusers/reusing_seeds
|
||||
title: "Reusing seeds for deterministic generation"
|
||||
title: Reusing seeds for deterministic generation
|
||||
- local: using-diffusers/custom_pipeline_examples
|
||||
title: "Community Pipelines"
|
||||
title: Community Pipelines
|
||||
- local: using-diffusers/contribute_pipeline
|
||||
title: "How to contribute a Pipeline"
|
||||
title: "Pipelines for Inference"
|
||||
title: How to contribute a Pipeline
|
||||
title: Pipelines for Inference
|
||||
- sections:
|
||||
- local: using-diffusers/rl
|
||||
title: "Reinforcement Learning"
|
||||
title: Reinforcement Learning
|
||||
- local: using-diffusers/audio
|
||||
title: "Audio"
|
||||
title: Audio
|
||||
- local: using-diffusers/other-modalities
|
||||
title: "Other Modalities"
|
||||
title: "Taking Diffusers Beyond Images"
|
||||
title: "Using Diffusers"
|
||||
title: Other Modalities
|
||||
title: Taking Diffusers Beyond Images
|
||||
title: Using Diffusers
|
||||
- sections:
|
||||
- local: optimization/fp16
|
||||
title: "Memory and Speed"
|
||||
title: Memory and Speed
|
||||
- local: optimization/xformers
|
||||
title: "xFormers"
|
||||
title: xFormers
|
||||
- local: optimization/onnx
|
||||
title: "ONNX"
|
||||
title: ONNX
|
||||
- local: optimization/open_vino
|
||||
title: "OpenVINO"
|
||||
title: OpenVINO
|
||||
- local: optimization/mps
|
||||
title: "MPS"
|
||||
title: MPS
|
||||
- local: optimization/habana
|
||||
title: "Habana Gaudi"
|
||||
title: "Optimization/Special Hardware"
|
||||
title: Habana Gaudi
|
||||
title: Optimization/Special Hardware
|
||||
- sections:
|
||||
- local: training/overview
|
||||
title: "Overview"
|
||||
title: Overview
|
||||
- local: training/unconditional_training
|
||||
title: "Unconditional Image Generation"
|
||||
title: Unconditional Image Generation
|
||||
- local: training/text_inversion
|
||||
title: "Textual Inversion"
|
||||
title: Textual Inversion
|
||||
- local: training/dreambooth
|
||||
title: "Dreambooth"
|
||||
title: Dreambooth
|
||||
- local: training/text2image
|
||||
title: "Text-to-image fine-tuning"
|
||||
title: "Training"
|
||||
title: Text-to-image fine-tuning
|
||||
title: Training
|
||||
- sections:
|
||||
- local: conceptual/philosophy
|
||||
title: "Philosophy"
|
||||
title: Philosophy
|
||||
- local: conceptual/contribution
|
||||
title: "How to contribute?"
|
||||
title: "Conceptual Guides"
|
||||
title: How to contribute?
|
||||
title: Conceptual Guides
|
||||
- sections:
|
||||
- sections:
|
||||
- local: api/models
|
||||
title: "Models"
|
||||
title: Models
|
||||
- local: api/diffusion_pipeline
|
||||
title: "Diffusion Pipeline"
|
||||
title: Diffusion Pipeline
|
||||
- local: api/logging
|
||||
title: "Logging"
|
||||
title: Logging
|
||||
- local: api/configuration
|
||||
title: "Configuration"
|
||||
title: Configuration
|
||||
- local: api/outputs
|
||||
title: "Outputs"
|
||||
title: "Main Classes"
|
||||
|
||||
title: Outputs
|
||||
title: Main Classes
|
||||
- sections:
|
||||
- local: api/pipelines/overview
|
||||
title: "Overview"
|
||||
title: Overview
|
||||
- local: api/pipelines/alt_diffusion
|
||||
title: "AltDiffusion"
|
||||
title: AltDiffusion
|
||||
- local: api/pipelines/audio_diffusion
|
||||
title: Audio Diffusion
|
||||
- local: api/pipelines/cycle_diffusion
|
||||
title: "Cycle Diffusion"
|
||||
title: Cycle Diffusion
|
||||
- local: api/pipelines/dance_diffusion
|
||||
title: Dance Diffusion
|
||||
- local: api/pipelines/ddim
|
||||
title: "DDIM"
|
||||
title: DDIM
|
||||
- local: api/pipelines/ddpm
|
||||
title: "DDPM"
|
||||
title: DDPM
|
||||
- local: api/pipelines/latent_diffusion
|
||||
title: "Latent Diffusion"
|
||||
- local: api/pipelines/latent_diffusion_uncond
|
||||
title: "Unconditional Latent Diffusion"
|
||||
title: Latent Diffusion
|
||||
- local: api/pipelines/paint_by_example
|
||||
title: "PaintByExample"
|
||||
title: PaintByExample
|
||||
- local: api/pipelines/pndm
|
||||
title: "PNDM"
|
||||
title: PNDM
|
||||
- local: api/pipelines/repaint
|
||||
title: RePaint
|
||||
- local: api/pipelines/stable_diffusion_safe
|
||||
title: Safe Stable Diffusion
|
||||
- local: api/pipelines/score_sde_ve
|
||||
title: "Score SDE VE"
|
||||
title: Score SDE VE
|
||||
- sections:
|
||||
- local: api/pipelines/stable_diffusion/overview
|
||||
title: "Overview"
|
||||
title: Overview
|
||||
- local: api/pipelines/stable_diffusion/text2img
|
||||
title: "Text-to-Image"
|
||||
title: Text-to-Image
|
||||
- local: api/pipelines/stable_diffusion/img2img
|
||||
title: "Image-to-Image"
|
||||
title: Image-to-Image
|
||||
- local: api/pipelines/stable_diffusion/inpaint
|
||||
title: "Inpaint"
|
||||
title: Inpaint
|
||||
- local: api/pipelines/stable_diffusion/depth2img
|
||||
title: "Depth-to-Image"
|
||||
title: Depth-to-Image
|
||||
- local: api/pipelines/stable_diffusion/image_variation
|
||||
title: "Image-Variation"
|
||||
title: Image-Variation
|
||||
- local: api/pipelines/stable_diffusion/upscale
|
||||
title: "Super-Resolution"
|
||||
title: "Stable Diffusion"
|
||||
title: Super-Resolution
|
||||
title: Stable Diffusion
|
||||
- local: api/pipelines/stable_diffusion_2
|
||||
title: "Stable Diffusion 2"
|
||||
- local: api/pipelines/stable_diffusion_safe
|
||||
title: "Safe Stable Diffusion"
|
||||
title: Stable Diffusion 2
|
||||
- local: api/pipelines/stochastic_karras_ve
|
||||
title: "Stochastic Karras VE"
|
||||
- local: api/pipelines/dance_diffusion
|
||||
title: "Dance Diffusion"
|
||||
title: Stochastic Karras VE
|
||||
- local: api/pipelines/unclip
|
||||
title: "UnCLIP"
|
||||
title: UnCLIP
|
||||
- local: api/pipelines/latent_diffusion_uncond
|
||||
title: Unconditional Latent Diffusion
|
||||
- local: api/pipelines/versatile_diffusion
|
||||
title: "Versatile Diffusion"
|
||||
title: Versatile Diffusion
|
||||
- local: api/pipelines/vq_diffusion
|
||||
title: "VQ Diffusion"
|
||||
- local: api/pipelines/repaint
|
||||
title: "RePaint"
|
||||
- local: api/pipelines/audio_diffusion
|
||||
title: "Audio Diffusion"
|
||||
title: "Pipelines"
|
||||
title: VQ Diffusion
|
||||
title: Pipelines
|
||||
- sections:
|
||||
- local: api/schedulers/overview
|
||||
title: "Overview"
|
||||
title: Overview
|
||||
- local: api/schedulers/ddim
|
||||
title: "DDIM"
|
||||
title: DDIM
|
||||
- local: api/schedulers/ddpm
|
||||
title: "DDPM"
|
||||
title: DDPM
|
||||
- local: api/schedulers/deis
|
||||
title: "DEIS"
|
||||
- local: api/schedulers/singlestep_dpm_solver
|
||||
title: "Singlestep DPM-Solver"
|
||||
- local: api/schedulers/multistep_dpm_solver
|
||||
title: "Multistep DPM-Solver"
|
||||
- local: api/schedulers/heun
|
||||
title: "Heun Scheduler"
|
||||
title: DEIS
|
||||
- local: api/schedulers/dpm_discrete
|
||||
title: "DPM Discrete Scheduler"
|
||||
title: DPM Discrete Scheduler
|
||||
- local: api/schedulers/dpm_discrete_ancestral
|
||||
title: "DPM Discrete Scheduler with ancestral sampling"
|
||||
- local: api/schedulers/stochastic_karras_ve
|
||||
title: "Stochastic Kerras VE"
|
||||
- local: api/schedulers/lms_discrete
|
||||
title: "Linear Multistep"
|
||||
- local: api/schedulers/pndm
|
||||
title: "PNDM"
|
||||
- local: api/schedulers/score_sde_ve
|
||||
title: "VE-SDE"
|
||||
- local: api/schedulers/ipndm
|
||||
title: "IPNDM"
|
||||
- local: api/schedulers/score_sde_vp
|
||||
title: "VP-SDE"
|
||||
- local: api/schedulers/euler
|
||||
title: "Euler scheduler"
|
||||
title: DPM Discrete Scheduler with ancestral sampling
|
||||
- local: api/schedulers/euler_ancestral
|
||||
title: "Euler Ancestral Scheduler"
|
||||
- local: api/schedulers/vq_diffusion
|
||||
title: "VQDiffusionScheduler"
|
||||
title: Euler Ancestral Scheduler
|
||||
- local: api/schedulers/euler
|
||||
title: Euler scheduler
|
||||
- local: api/schedulers/heun
|
||||
title: Heun Scheduler
|
||||
- local: api/schedulers/ipndm
|
||||
title: IPNDM
|
||||
- local: api/schedulers/lms_discrete
|
||||
title: Linear Multistep
|
||||
- local: api/schedulers/multistep_dpm_solver
|
||||
title: Multistep DPM-Solver
|
||||
- local: api/schedulers/pndm
|
||||
title: PNDM
|
||||
- local: api/schedulers/repaint
|
||||
title: "RePaint Scheduler"
|
||||
title: "Schedulers"
|
||||
title: RePaint Scheduler
|
||||
- local: api/schedulers/singlestep_dpm_solver
|
||||
title: Singlestep DPM-Solver
|
||||
- local: api/schedulers/stochastic_karras_ve
|
||||
title: Stochastic Kerras VE
|
||||
- local: api/schedulers/score_sde_ve
|
||||
title: VE-SDE
|
||||
- local: api/schedulers/score_sde_vp
|
||||
title: VP-SDE
|
||||
- local: api/schedulers/vq_diffusion
|
||||
title: VQDiffusionScheduler
|
||||
title: Schedulers
|
||||
- sections:
|
||||
- local: api/experimental/rl
|
||||
title: "RL Planning"
|
||||
title: "Experimental Features"
|
||||
title: "API"
|
||||
title: RL Planning
|
||||
title: Experimental Features
|
||||
title: API
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
# coding=utf-8
|
||||
# Copyright 2022 The HuggingFace Inc. team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
PATH_TO_TOC = "docs/source/en/_toctree.yml"
|
||||
|
||||
|
||||
def clean_doc_toc(doc_list):
|
||||
"""
|
||||
Cleans the table of content of the model documentation by removing duplicates and sorting models alphabetically.
|
||||
"""
|
||||
counts = defaultdict(int)
|
||||
overview_doc = []
|
||||
new_doc_list = []
|
||||
for doc in doc_list:
|
||||
if "local" in doc:
|
||||
counts[doc["local"]] += 1
|
||||
|
||||
if doc["title"].lower() == "overview":
|
||||
overview_doc.append({"local": doc["local"], "title": doc["title"]})
|
||||
else:
|
||||
new_doc_list.append(doc)
|
||||
|
||||
doc_list = new_doc_list
|
||||
duplicates = [key for key, value in counts.items() if value > 1]
|
||||
|
||||
new_doc = []
|
||||
for duplicate_key in duplicates:
|
||||
titles = list(set(doc["title"] for doc in doc_list if doc["local"] == duplicate_key))
|
||||
if len(titles) > 1:
|
||||
raise ValueError(
|
||||
f"{duplicate_key} is present several times in the documentation table of content at "
|
||||
"`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the "
|
||||
"others."
|
||||
)
|
||||
# Only add this once
|
||||
new_doc.append({"local": duplicate_key, "title": titles[0]})
|
||||
|
||||
# Add none duplicate-keys
|
||||
new_doc.extend([doc for doc in doc_list if "local" not in counts or counts[doc["local"]] == 1])
|
||||
new_doc = sorted(new_doc, key=lambda s: s["title"].lower())
|
||||
|
||||
# "overview" gets special treatment and is always first
|
||||
if len(overview_doc) > 1:
|
||||
raise ValueError("{doc_list} has two 'overview' docs which is not allowed.")
|
||||
|
||||
overview_doc.extend(new_doc)
|
||||
|
||||
# Sort
|
||||
return overview_doc
|
||||
|
||||
|
||||
def check_scheduler_doc(overwrite=False):
|
||||
with open(PATH_TO_TOC, encoding="utf-8") as f:
|
||||
content = yaml.safe_load(f.read())
|
||||
|
||||
# Get to the API doc
|
||||
api_idx = 0
|
||||
while content[api_idx]["title"] != "API":
|
||||
api_idx += 1
|
||||
api_doc = content[api_idx]["sections"]
|
||||
|
||||
# Then to the model doc
|
||||
scheduler_idx = 0
|
||||
while api_doc[scheduler_idx]["title"] != "Schedulers":
|
||||
scheduler_idx += 1
|
||||
|
||||
scheduler_doc = api_doc[scheduler_idx]["sections"]
|
||||
new_scheduler_doc = clean_doc_toc(scheduler_doc)
|
||||
|
||||
diff = False
|
||||
if new_scheduler_doc != scheduler_doc:
|
||||
diff = True
|
||||
if overwrite:
|
||||
api_doc[scheduler_idx]["sections"] = new_scheduler_doc
|
||||
|
||||
if diff:
|
||||
if overwrite:
|
||||
content[api_idx]["sections"] = api_doc
|
||||
with open(PATH_TO_TOC, "w", encoding="utf-8") as f:
|
||||
f.write(yaml.dump(content, allow_unicode=True))
|
||||
else:
|
||||
raise ValueError(
|
||||
"The model doc part of the table of content is not properly sorted, run `make style` to fix this."
|
||||
)
|
||||
|
||||
|
||||
def check_pipeline_doc(overwrite=False):
|
||||
with open(PATH_TO_TOC, encoding="utf-8") as f:
|
||||
content = yaml.safe_load(f.read())
|
||||
|
||||
# Get to the API doc
|
||||
api_idx = 0
|
||||
while content[api_idx]["title"] != "API":
|
||||
api_idx += 1
|
||||
api_doc = content[api_idx]["sections"]
|
||||
|
||||
# Then to the model doc
|
||||
pipeline_idx = 0
|
||||
while api_doc[pipeline_idx]["title"] != "Pipelines":
|
||||
pipeline_idx += 1
|
||||
|
||||
diff = False
|
||||
pipeline_docs = api_doc[pipeline_idx]["sections"]
|
||||
new_pipeline_docs = []
|
||||
|
||||
# sort sub pipeline docs
|
||||
for pipeline_doc in pipeline_docs:
|
||||
if "section" in pipeline_doc:
|
||||
sub_pipeline_doc = pipeline_doc["section"]
|
||||
new_sub_pipeline_doc = clean_doc_toc(sub_pipeline_doc)
|
||||
if overwrite:
|
||||
pipeline_doc["section"] = new_sub_pipeline_doc
|
||||
new_pipeline_docs.append(pipeline_doc)
|
||||
|
||||
# sort overall pipeline doc
|
||||
new_pipeline_docs = clean_doc_toc(new_pipeline_docs)
|
||||
|
||||
if new_pipeline_docs != pipeline_docs:
|
||||
diff = True
|
||||
if overwrite:
|
||||
api_doc[pipeline_idx]["sections"] = new_pipeline_docs
|
||||
|
||||
if diff:
|
||||
if overwrite:
|
||||
content[api_idx]["sections"] = api_doc
|
||||
with open(PATH_TO_TOC, "w", encoding="utf-8") as f:
|
||||
f.write(yaml.dump(content, allow_unicode=True))
|
||||
else:
|
||||
raise ValueError(
|
||||
"The model doc part of the table of content is not properly sorted, run `make style` to fix this."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
|
||||
args = parser.parse_args()
|
||||
|
||||
check_scheduler_doc(args.fix_and_overwrite)
|
||||
check_pipeline_doc(args.fix_and_overwrite)
|
||||
Loading…
Reference in New Issue