diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..4b33fe187b --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,52 @@ +name: Deploy docs +on: + workflow_dispatch: + push: + branches: + - 'master' + pull_request: +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + if: (github.event_name != 'pull_request') + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: | + setup.py + requirements-docs.txt + + - name: Save time for cache for mkdocs + run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + + - name: Caching + uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + + - name: Install Dependencies + run: pip install -r requirements-docs.txt + + - name: Deploy to GitHub Pages + run: mkdocs gh-deploy --force + if: (github.event_name != 'pull_request') + + - name: Build docs to check for errors + run: mkdocs build + if: (github.event_name == 'pull_request') diff --git a/g3doc/_toc.yaml b/docs/_toc.yaml similarity index 100% rename from g3doc/_toc.yaml rename to docs/_toc.yaml diff --git a/g3doc/api_docs/python/_redirects.yaml b/docs/api_docs/python/deprecated/_redirects.yaml similarity index 100% rename from g3doc/api_docs/python/_redirects.yaml rename to docs/api_docs/python/deprecated/_redirects.yaml diff --git a/g3doc/api_docs/python/_toc.yaml b/docs/api_docs/python/deprecated/_toc.yaml similarity index 100% rename from g3doc/api_docs/python/_toc.yaml rename to docs/api_docs/python/deprecated/_toc.yaml diff --git a/g3doc/api_docs/python/index.md b/docs/api_docs/python/deprecated/index.md similarity index 100% rename from g3doc/api_docs/python/index.md rename to docs/api_docs/python/deprecated/index.md diff --git a/g3doc/api_docs/python/tfma.md b/docs/api_docs/python/deprecated/tfma.md similarity index 100% rename from g3doc/api_docs/python/tfma.md rename to docs/api_docs/python/deprecated/tfma.md diff --git a/g3doc/api_docs/python/tfma/EvalConfig.md b/docs/api_docs/python/deprecated/tfma/EvalConfig.md similarity index 100% rename from g3doc/api_docs/python/tfma/EvalConfig.md rename to docs/api_docs/python/deprecated/tfma/EvalConfig.md diff --git a/g3doc/api_docs/python/tfma/EvalResult.md b/docs/api_docs/python/deprecated/tfma/EvalResult.md similarity index 100% rename from g3doc/api_docs/python/tfma/EvalResult.md rename to docs/api_docs/python/deprecated/tfma/EvalResult.md diff --git a/g3doc/api_docs/python/tfma/ExtractAndEvaluate.md b/docs/api_docs/python/deprecated/tfma/ExtractAndEvaluate.md similarity index 100% rename from g3doc/api_docs/python/tfma/ExtractAndEvaluate.md rename to docs/api_docs/python/deprecated/tfma/ExtractAndEvaluate.md diff --git a/g3doc/api_docs/python/tfma/ExtractEvaluateAndWriteResults.md b/docs/api_docs/python/deprecated/tfma/ExtractEvaluateAndWriteResults.md similarity index 100% rename from g3doc/api_docs/python/tfma/ExtractEvaluateAndWriteResults.md rename to docs/api_docs/python/deprecated/tfma/ExtractEvaluateAndWriteResults.md diff --git a/g3doc/api_docs/python/tfma/InputsToExtracts.md b/docs/api_docs/python/deprecated/tfma/InputsToExtracts.md similarity index 100% rename from g3doc/api_docs/python/tfma/InputsToExtracts.md rename to docs/api_docs/python/deprecated/tfma/InputsToExtracts.md diff --git a/g3doc/api_docs/python/tfma/Validate.md b/docs/api_docs/python/deprecated/tfma/Validate.md similarity index 100% rename from g3doc/api_docs/python/tfma/Validate.md rename to docs/api_docs/python/deprecated/tfma/Validate.md diff --git a/g3doc/api_docs/python/tfma/WriteResults.md b/docs/api_docs/python/deprecated/tfma/WriteResults.md similarity index 100% rename from g3doc/api_docs/python/tfma/WriteResults.md rename to docs/api_docs/python/deprecated/tfma/WriteResults.md diff --git a/g3doc/api_docs/python/tfma/compound_key.md b/docs/api_docs/python/deprecated/tfma/compound_key.md similarity index 100% rename from g3doc/api_docs/python/tfma/compound_key.md rename to docs/api_docs/python/deprecated/tfma/compound_key.md diff --git a/g3doc/api_docs/python/tfma/constants.md b/docs/api_docs/python/deprecated/tfma/constants.md similarity index 100% rename from g3doc/api_docs/python/tfma/constants.md rename to docs/api_docs/python/deprecated/tfma/constants.md diff --git a/g3doc/api_docs/python/tfma/create_keys_key.md b/docs/api_docs/python/deprecated/tfma/create_keys_key.md similarity index 100% rename from g3doc/api_docs/python/tfma/create_keys_key.md rename to docs/api_docs/python/deprecated/tfma/create_keys_key.md diff --git a/g3doc/api_docs/python/tfma/create_values_key.md b/docs/api_docs/python/deprecated/tfma/create_values_key.md similarity index 100% rename from g3doc/api_docs/python/tfma/create_values_key.md rename to docs/api_docs/python/deprecated/tfma/create_values_key.md diff --git a/g3doc/api_docs/python/tfma/default_eval_shared_model.md b/docs/api_docs/python/deprecated/tfma/default_eval_shared_model.md similarity index 100% rename from g3doc/api_docs/python/tfma/default_eval_shared_model.md rename to docs/api_docs/python/deprecated/tfma/default_eval_shared_model.md diff --git a/g3doc/api_docs/python/tfma/default_evaluators.md b/docs/api_docs/python/deprecated/tfma/default_evaluators.md similarity index 100% rename from g3doc/api_docs/python/tfma/default_evaluators.md rename to docs/api_docs/python/deprecated/tfma/default_evaluators.md diff --git a/g3doc/api_docs/python/tfma/default_extractors.md b/docs/api_docs/python/deprecated/tfma/default_extractors.md similarity index 100% rename from g3doc/api_docs/python/tfma/default_extractors.md rename to docs/api_docs/python/deprecated/tfma/default_extractors.md diff --git a/g3doc/api_docs/python/tfma/default_writers.md b/docs/api_docs/python/deprecated/tfma/default_writers.md similarity index 100% rename from g3doc/api_docs/python/tfma/default_writers.md rename to docs/api_docs/python/deprecated/tfma/default_writers.md diff --git a/g3doc/api_docs/python/tfma/evaluators.md b/docs/api_docs/python/deprecated/tfma/evaluators.md similarity index 100% rename from g3doc/api_docs/python/tfma/evaluators.md rename to docs/api_docs/python/deprecated/tfma/evaluators.md diff --git a/g3doc/api_docs/python/tfma/evaluators/AnalysisTableEvaluator.md b/docs/api_docs/python/deprecated/tfma/evaluators/AnalysisTableEvaluator.md similarity index 100% rename from g3doc/api_docs/python/tfma/evaluators/AnalysisTableEvaluator.md rename to docs/api_docs/python/deprecated/tfma/evaluators/AnalysisTableEvaluator.md diff --git a/g3doc/api_docs/python/tfma/evaluators/Evaluator.md b/docs/api_docs/python/deprecated/tfma/evaluators/Evaluator.md similarity index 100% rename from g3doc/api_docs/python/tfma/evaluators/Evaluator.md rename to docs/api_docs/python/deprecated/tfma/evaluators/Evaluator.md diff --git a/g3doc/api_docs/python/tfma/evaluators/MetricsAndPlotsEvaluator.md b/docs/api_docs/python/deprecated/tfma/evaluators/MetricsAndPlotsEvaluator.md similarity index 100% rename from g3doc/api_docs/python/tfma/evaluators/MetricsAndPlotsEvaluator.md rename to docs/api_docs/python/deprecated/tfma/evaluators/MetricsAndPlotsEvaluator.md diff --git a/g3doc/api_docs/python/tfma/evaluators/verify_evaluator.md b/docs/api_docs/python/deprecated/tfma/evaluators/verify_evaluator.md similarity index 100% rename from g3doc/api_docs/python/tfma/evaluators/verify_evaluator.md rename to docs/api_docs/python/deprecated/tfma/evaluators/verify_evaluator.md diff --git a/g3doc/api_docs/python/tfma/export.md b/docs/api_docs/python/deprecated/tfma/export.md similarity index 100% rename from g3doc/api_docs/python/tfma/export.md rename to docs/api_docs/python/deprecated/tfma/export.md diff --git a/g3doc/api_docs/python/tfma/export/build_parsing_eval_input_receiver_fn.md b/docs/api_docs/python/deprecated/tfma/export/build_parsing_eval_input_receiver_fn.md similarity index 100% rename from g3doc/api_docs/python/tfma/export/build_parsing_eval_input_receiver_fn.md rename to docs/api_docs/python/deprecated/tfma/export/build_parsing_eval_input_receiver_fn.md diff --git a/g3doc/api_docs/python/tfma/exporter.md b/docs/api_docs/python/deprecated/tfma/exporter.md similarity index 100% rename from g3doc/api_docs/python/tfma/exporter.md rename to docs/api_docs/python/deprecated/tfma/exporter.md diff --git a/g3doc/api_docs/python/tfma/exporter/FinalExporter.md b/docs/api_docs/python/deprecated/tfma/exporter/FinalExporter.md similarity index 100% rename from g3doc/api_docs/python/tfma/exporter/FinalExporter.md rename to docs/api_docs/python/deprecated/tfma/exporter/FinalExporter.md diff --git a/g3doc/api_docs/python/tfma/exporter/LatestExporter.md b/docs/api_docs/python/deprecated/tfma/exporter/LatestExporter.md similarity index 100% rename from g3doc/api_docs/python/tfma/exporter/LatestExporter.md rename to docs/api_docs/python/deprecated/tfma/exporter/LatestExporter.md diff --git a/g3doc/api_docs/python/tfma/extractors.md b/docs/api_docs/python/deprecated/tfma/extractors.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors.md rename to docs/api_docs/python/deprecated/tfma/extractors.md diff --git a/g3doc/api_docs/python/tfma/extractors/Extractor.md b/docs/api_docs/python/deprecated/tfma/extractors/Extractor.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors/Extractor.md rename to docs/api_docs/python/deprecated/tfma/extractors/Extractor.md diff --git a/g3doc/api_docs/python/tfma/extractors/FeatureExtractor.md b/docs/api_docs/python/deprecated/tfma/extractors/FeatureExtractor.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors/FeatureExtractor.md rename to docs/api_docs/python/deprecated/tfma/extractors/FeatureExtractor.md diff --git a/g3doc/api_docs/python/tfma/extractors/Filter.md b/docs/api_docs/python/deprecated/tfma/extractors/Filter.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors/Filter.md rename to docs/api_docs/python/deprecated/tfma/extractors/Filter.md diff --git a/g3doc/api_docs/python/tfma/extractors/PredictExtractor.md b/docs/api_docs/python/deprecated/tfma/extractors/PredictExtractor.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors/PredictExtractor.md rename to docs/api_docs/python/deprecated/tfma/extractors/PredictExtractor.md diff --git a/g3doc/api_docs/python/tfma/extractors/SliceKeyExtractor.md b/docs/api_docs/python/deprecated/tfma/extractors/SliceKeyExtractor.md similarity index 100% rename from g3doc/api_docs/python/tfma/extractors/SliceKeyExtractor.md rename to docs/api_docs/python/deprecated/tfma/extractors/SliceKeyExtractor.md diff --git a/g3doc/api_docs/python/tfma/load_eval_result.md b/docs/api_docs/python/deprecated/tfma/load_eval_result.md similarity index 100% rename from g3doc/api_docs/python/tfma/load_eval_result.md rename to docs/api_docs/python/deprecated/tfma/load_eval_result.md diff --git a/g3doc/api_docs/python/tfma/load_eval_results.md b/docs/api_docs/python/deprecated/tfma/load_eval_results.md similarity index 100% rename from g3doc/api_docs/python/tfma/load_eval_results.md rename to docs/api_docs/python/deprecated/tfma/load_eval_results.md diff --git a/g3doc/api_docs/python/tfma/make_eval_results.md b/docs/api_docs/python/deprecated/tfma/make_eval_results.md similarity index 100% rename from g3doc/api_docs/python/tfma/make_eval_results.md rename to docs/api_docs/python/deprecated/tfma/make_eval_results.md diff --git a/g3doc/api_docs/python/tfma/multiple_data_analysis.md b/docs/api_docs/python/deprecated/tfma/multiple_data_analysis.md similarity index 100% rename from g3doc/api_docs/python/tfma/multiple_data_analysis.md rename to docs/api_docs/python/deprecated/tfma/multiple_data_analysis.md diff --git a/g3doc/api_docs/python/tfma/multiple_model_analysis.md b/docs/api_docs/python/deprecated/tfma/multiple_model_analysis.md similarity index 100% rename from g3doc/api_docs/python/tfma/multiple_model_analysis.md rename to docs/api_docs/python/deprecated/tfma/multiple_model_analysis.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/auc.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/auc.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/auc.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/auc.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/auc_plots.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/auc_plots.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/auc_plots.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/auc_plots.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/calibration_plot_and_prediction_histogram.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/calibration_plot_and_prediction_histogram.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/calibration_plot_and_prediction_histogram.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/calibration_plot_and_prediction_histogram.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/confusion_matrix_at_thresholds.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/confusion_matrix_at_thresholds.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/confusion_matrix_at_thresholds.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/confusion_matrix_at_thresholds.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/example_count.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/example_count.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/example_count.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/example_count.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/example_weight.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/example_weight.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/example_weight.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/example_weight.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/mean_absolute_error.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/mean_absolute_error.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/mean_absolute_error.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/mean_absolute_error.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/precision_at_k.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/precision_at_k.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/precision_at_k.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/precision_at_k.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/recall_at_k.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/recall_at_k.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/recall_at_k.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/recall_at_k.md diff --git a/g3doc/api_docs/python/tfma/post_export_metrics/squared_pearson_correlation.md b/docs/api_docs/python/deprecated/tfma/post_export_metrics/squared_pearson_correlation.md similarity index 100% rename from g3doc/api_docs/python/tfma/post_export_metrics/squared_pearson_correlation.md rename to docs/api_docs/python/deprecated/tfma/post_export_metrics/squared_pearson_correlation.md diff --git a/g3doc/api_docs/python/tfma/run_model_analysis.md b/docs/api_docs/python/deprecated/tfma/run_model_analysis.md similarity index 100% rename from g3doc/api_docs/python/tfma/run_model_analysis.md rename to docs/api_docs/python/deprecated/tfma/run_model_analysis.md diff --git a/g3doc/api_docs/python/tfma/types.md b/docs/api_docs/python/deprecated/tfma/types.md similarity index 100% rename from g3doc/api_docs/python/tfma/types.md rename to docs/api_docs/python/deprecated/tfma/types.md diff --git a/g3doc/api_docs/python/tfma/types/EvalSharedModel.md b/docs/api_docs/python/deprecated/tfma/types/EvalSharedModel.md similarity index 100% rename from g3doc/api_docs/python/tfma/types/EvalSharedModel.md rename to docs/api_docs/python/deprecated/tfma/types/EvalSharedModel.md diff --git a/g3doc/api_docs/python/tfma/types/FeaturesPredictionsLabels.md b/docs/api_docs/python/deprecated/tfma/types/FeaturesPredictionsLabels.md similarity index 100% rename from g3doc/api_docs/python/tfma/types/FeaturesPredictionsLabels.md rename to docs/api_docs/python/deprecated/tfma/types/FeaturesPredictionsLabels.md diff --git a/g3doc/api_docs/python/tfma/types/MaterializedColumn.md b/docs/api_docs/python/deprecated/tfma/types/MaterializedColumn.md similarity index 100% rename from g3doc/api_docs/python/tfma/types/MaterializedColumn.md rename to docs/api_docs/python/deprecated/tfma/types/MaterializedColumn.md diff --git a/g3doc/api_docs/python/tfma/types/ValueWithConfidenceInterval.md b/docs/api_docs/python/deprecated/tfma/types/ValueWithConfidenceInterval.md similarity index 100% rename from g3doc/api_docs/python/tfma/types/ValueWithConfidenceInterval.md rename to docs/api_docs/python/deprecated/tfma/types/ValueWithConfidenceInterval.md diff --git a/g3doc/api_docs/python/tfma/types/is_tensor.md b/docs/api_docs/python/deprecated/tfma/types/is_tensor.md similarity index 100% rename from g3doc/api_docs/python/tfma/types/is_tensor.md rename to docs/api_docs/python/deprecated/tfma/types/is_tensor.md diff --git a/g3doc/api_docs/python/tfma/unique_key.md b/docs/api_docs/python/deprecated/tfma/unique_key.md similarity index 100% rename from g3doc/api_docs/python/tfma/unique_key.md rename to docs/api_docs/python/deprecated/tfma/unique_key.md diff --git a/g3doc/api_docs/python/tfma/validators.md b/docs/api_docs/python/deprecated/tfma/validators.md similarity index 100% rename from g3doc/api_docs/python/tfma/validators.md rename to docs/api_docs/python/deprecated/tfma/validators.md diff --git a/g3doc/api_docs/python/tfma/validators/Validator.md b/docs/api_docs/python/deprecated/tfma/validators/Validator.md similarity index 100% rename from g3doc/api_docs/python/tfma/validators/Validator.md rename to docs/api_docs/python/deprecated/tfma/validators/Validator.md diff --git a/g3doc/api_docs/python/tfma/version.md b/docs/api_docs/python/deprecated/tfma/version.md similarity index 100% rename from g3doc/api_docs/python/tfma/version.md rename to docs/api_docs/python/deprecated/tfma/version.md diff --git a/g3doc/api_docs/python/tfma/view.md b/docs/api_docs/python/deprecated/tfma/view.md similarity index 100% rename from g3doc/api_docs/python/tfma/view.md rename to docs/api_docs/python/deprecated/tfma/view.md diff --git a/g3doc/api_docs/python/tfma/view/render_plot.md b/docs/api_docs/python/deprecated/tfma/view/render_plot.md similarity index 100% rename from g3doc/api_docs/python/tfma/view/render_plot.md rename to docs/api_docs/python/deprecated/tfma/view/render_plot.md diff --git a/g3doc/api_docs/python/tfma/view/render_slicing_metrics.md b/docs/api_docs/python/deprecated/tfma/view/render_slicing_metrics.md similarity index 100% rename from g3doc/api_docs/python/tfma/view/render_slicing_metrics.md rename to docs/api_docs/python/deprecated/tfma/view/render_slicing_metrics.md diff --git a/g3doc/api_docs/python/tfma/view/render_time_series.md b/docs/api_docs/python/deprecated/tfma/view/render_time_series.md similarity index 100% rename from g3doc/api_docs/python/tfma/view/render_time_series.md rename to docs/api_docs/python/deprecated/tfma/view/render_time_series.md diff --git a/g3doc/api_docs/python/tfma/writers.md b/docs/api_docs/python/deprecated/tfma/writers.md similarity index 100% rename from g3doc/api_docs/python/tfma/writers.md rename to docs/api_docs/python/deprecated/tfma/writers.md diff --git a/g3doc/api_docs/python/tfma/writers/Write.md b/docs/api_docs/python/deprecated/tfma/writers/Write.md similarity index 100% rename from g3doc/api_docs/python/tfma/writers/Write.md rename to docs/api_docs/python/deprecated/tfma/writers/Write.md diff --git a/g3doc/api_docs/python/tfma/writers/Writer.md b/docs/api_docs/python/deprecated/tfma/writers/Writer.md similarity index 100% rename from g3doc/api_docs/python/tfma/writers/Writer.md rename to docs/api_docs/python/deprecated/tfma/writers/Writer.md diff --git a/docs/api_docs/python/tfma-constants.md b/docs/api_docs/python/tfma-constants.md new file mode 100644 index 0000000000..347c58dfbc --- /dev/null +++ b/docs/api_docs/python/tfma-constants.md @@ -0,0 +1,3 @@ +# TFMA Constants + +::: tensorflow_model_analysis.constants diff --git a/docs/api_docs/python/tfma-contrib.md b/docs/api_docs/python/tfma-contrib.md new file mode 100644 index 0000000000..12283ff0c7 --- /dev/null +++ b/docs/api_docs/python/tfma-contrib.md @@ -0,0 +1,3 @@ +# TFMA Constants + +::: tensorflow_model_analysis.contrib diff --git a/docs/api_docs/python/tfma-evaluators.md b/docs/api_docs/python/tfma-evaluators.md new file mode 100644 index 0000000000..bb098cfd48 --- /dev/null +++ b/docs/api_docs/python/tfma-evaluators.md @@ -0,0 +1,5 @@ + +# TFMA Evaluators + +::: tensorflow_model_analysis.evaluators + diff --git a/docs/api_docs/python/tfma-experimental.md b/docs/api_docs/python/tfma-experimental.md new file mode 100644 index 0000000000..77af1ee691 --- /dev/null +++ b/docs/api_docs/python/tfma-experimental.md @@ -0,0 +1,5 @@ + +# TFMA Experimental + +::: tensorflow_model_analysis.experimental + diff --git a/docs/api_docs/python/tfma-extractors.md b/docs/api_docs/python/tfma-extractors.md new file mode 100644 index 0000000000..08ab4f7263 --- /dev/null +++ b/docs/api_docs/python/tfma-extractors.md @@ -0,0 +1,5 @@ + +# TFMA Extractors + +::: tensorflow_model_analysis.extractors + diff --git a/docs/api_docs/python/tfma-metrics.md b/docs/api_docs/python/tfma-metrics.md new file mode 100644 index 0000000000..0e424df164 --- /dev/null +++ b/docs/api_docs/python/tfma-metrics.md @@ -0,0 +1,5 @@ + +# TFMA Metrics + +::: tensorflow_model_analysis.metrics + diff --git a/docs/api_docs/python/tfma-post_export_metrics.md b/docs/api_docs/python/tfma-post_export_metrics.md new file mode 100644 index 0000000000..c476e10ca1 --- /dev/null +++ b/docs/api_docs/python/tfma-post_export_metrics.md @@ -0,0 +1,5 @@ + +# TFMA Post_Export_Metrics + +::: tensorflow_model_analysis.post_export_metrics + diff --git a/docs/api_docs/python/tfma-sdk.md b/docs/api_docs/python/tfma-sdk.md new file mode 100644 index 0000000000..267fd7d780 --- /dev/null +++ b/docs/api_docs/python/tfma-sdk.md @@ -0,0 +1,5 @@ + +# TFMA SDK + +::: tensorflow_model_analysis.sdk + diff --git a/docs/api_docs/python/tfma-types.md b/docs/api_docs/python/tfma-types.md new file mode 100644 index 0000000000..4b859da9cb --- /dev/null +++ b/docs/api_docs/python/tfma-types.md @@ -0,0 +1,5 @@ + +# TFMA Types + +::: tensorflow_model_analysis.types + diff --git a/docs/api_docs/python/tfma-utils.md b/docs/api_docs/python/tfma-utils.md new file mode 100644 index 0000000000..6057d9e977 --- /dev/null +++ b/docs/api_docs/python/tfma-utils.md @@ -0,0 +1,5 @@ + +# TFMA Utils + +::: tensorflow_model_analysis.utils + diff --git a/docs/api_docs/python/tfma-validators.md b/docs/api_docs/python/tfma-validators.md new file mode 100644 index 0000000000..835d8f55c8 --- /dev/null +++ b/docs/api_docs/python/tfma-validators.md @@ -0,0 +1,5 @@ + +# TFMA Validators + +::: tensorflow_model_analysis.validators + diff --git a/docs/api_docs/python/tfma-version.md b/docs/api_docs/python/tfma-version.md new file mode 100644 index 0000000000..fe4a03fbe4 --- /dev/null +++ b/docs/api_docs/python/tfma-version.md @@ -0,0 +1,5 @@ + +# TFMA Version + +::: tensorflow_model_analysis.version + diff --git a/docs/api_docs/python/tfma-view.md b/docs/api_docs/python/tfma-view.md new file mode 100644 index 0000000000..935553d98d --- /dev/null +++ b/docs/api_docs/python/tfma-view.md @@ -0,0 +1,5 @@ + +# TFMA View + +::: tensorflow_model_analysis.view + diff --git a/docs/api_docs/python/tfma-writers.md b/docs/api_docs/python/tfma-writers.md new file mode 100644 index 0000000000..26ca97058c --- /dev/null +++ b/docs/api_docs/python/tfma-writers.md @@ -0,0 +1,5 @@ + +# TFMA Writers + +::: tensorflow_model_analysis.writers + diff --git a/docs/api_docs/python/tfma.md b/docs/api_docs/python/tfma.md new file mode 100644 index 0000000000..4c6948c706 --- /dev/null +++ b/docs/api_docs/python/tfma.md @@ -0,0 +1,3 @@ +# TFMA + +::: tensorflow_model_analysis diff --git a/g3doc/architecture.md b/docs/architecture.md similarity index 100% rename from g3doc/architecture.md rename to docs/architecture.md diff --git a/g3doc/eval_saved_model.md b/docs/eval_saved_model.md similarity index 100% rename from g3doc/eval_saved_model.md rename to docs/eval_saved_model.md diff --git a/g3doc/faq.md b/docs/faq.md similarity index 98% rename from g3doc/faq.md rename to docs/faq.md index 6b8c0b8f85..d07fde6bee 100644 --- a/g3doc/faq.md +++ b/docs/faq.md @@ -49,7 +49,7 @@ TFMA supports keras models, models based on generic TF2 signature APIs, as well TF estimator based models (although depending on the use case the estimator based models may require an `EvalSavedModel` to be used). -See [get_started](get_started.md) guide for the full list of model types +See [get started](get_started.md) guide for the full list of model types supported and any restrictions. ### How do I setup TFMA to work with a native keras based model? @@ -61,7 +61,7 @@ assumptions: (this can be changed using `model_specs[0].signature_name`). * Built in metrics from `model.compile(...)` should be evaluated (this can be disabled via `options.include_default_metric` within the - [tfma.EvalConfig](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/EvalConfig)). + [tfma.EvalConfig](../api_docs/python/tfma#tensorflow_model_analysis.EvalConfig)). ```python from google.protobuf import text_format @@ -425,7 +425,7 @@ single label per example, not for multi-label. This is most likely caused because the metrics are configured for a binary classification problem, but the model is outputing probabilities for both of the classes instead of just one. This is common when -[tensorflow's classification API](https://www.tensorflow.org/tfx/serving/signature_defs#classification_signaturedef) +[tensorflow's classification API](https://tensorflow.github.io/serving/guide/signature_defs/#classification-signaturedef) is used. The solution is to choose the class that you would like the predictions to be based on and then binarize on that class. For example: diff --git a/g3doc/get_started.md b/docs/get_started.md similarity index 94% rename from g3doc/get_started.md rename to docs/get_started.md index fac157a5b4..2ff4258e52 100644 --- a/g3doc/get_started.md +++ b/docs/get_started.md @@ -25,7 +25,7 @@ If you just want to jump in and get started, check out our notebook. This page can also be viewed from -[tensorflow.org](https://www.tensorflow.org/tfx/model_analysis/get_started). +[tensorflow.org](https://tensorflow.github.io/model-analysis/get_started). ## Model Types Supported @@ -75,19 +75,19 @@ different model types. ## Setup Before running an evaluation, a small amount of setup is required. First, a -[`tfma.EvalConfig`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/EvalConfig) +[`tfma.EvalConfig`](../api_docs/python/tfma#tensorflow_model_analysis.EvalConfig) object must be defined that provides specifications for the model, metrics, and slices that are to be evaluated. Second a -[`tfma.EvalSharedModel`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/default_eval_shared_model) +[`tfma.EvalSharedModel`](../api_docs/python/tfma#tensorflow_model_analysis.default_eval_shared_model) needs to be created that points to the actual model (or models) to be used during the evaluation. Once these have been defined, evaluation is performed by calling -[`tfma.run_model_analysis`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/run_model_analysis) +[`tfma.run_model_analysis`](../api_docs/python/tfma#tensorflow_model_analysis.run_model_analysis) with an appropriate dataset. For more details, see the [setup](setup.md) guide. If running within a TFX pipeline, see the TFX -[guide](https://www.tensorflow.org/tfx/guide) for how to configure TFMA to run -as a TFX [Evaluator](https://www.tensorflow.org/tfx/guide/evaluator) component. +[guide](https://tensorflow.github.io/tfx/guide) for how to configure TFMA to run +as a TFX [Evaluator](https://tensorflow.github.io/tfx/guide/evaluator) component. ## Examples @@ -296,4 +296,4 @@ frontend components included in TFMA. For example: * [Visualizations](visualizations.md) * [Architecture](architecture.md) * [FAQ](faq.md) -* [API Reference](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma) +* [API Reference](../api_docs/python/tfma) diff --git a/g3doc/images/metricsHistogram.png b/docs/images/metricsHistogram.png similarity index 100% rename from g3doc/images/metricsHistogram.png rename to docs/images/metricsHistogram.png diff --git a/g3doc/images/metricsHistogramFiltered.png b/docs/images/metricsHistogramFiltered.png similarity index 100% rename from g3doc/images/metricsHistogramFiltered.png rename to docs/images/metricsHistogramFiltered.png diff --git a/g3doc/images/metricsHistogramSetting.png b/docs/images/metricsHistogramSetting.png similarity index 100% rename from g3doc/images/metricsHistogramSetting.png rename to docs/images/metricsHistogramSetting.png diff --git a/g3doc/images/metricsSelector.png b/docs/images/metricsSelector.png similarity index 100% rename from g3doc/images/metricsSelector.png rename to docs/images/metricsSelector.png diff --git a/g3doc/images/modelDrivenTimeSeriesGraph.png b/docs/images/modelDrivenTimeSeriesGraph.png similarity index 100% rename from g3doc/images/modelDrivenTimeSeriesGraph.png rename to docs/images/modelDrivenTimeSeriesGraph.png diff --git a/g3doc/images/multi_class_vs_binarized_metric_calculations.png b/docs/images/multi_class_vs_binarized_metric_calculations.png similarity index 100% rename from g3doc/images/multi_class_vs_binarized_metric_calculations.png rename to docs/images/multi_class_vs_binarized_metric_calculations.png diff --git a/g3doc/images/sliceOverview.png b/docs/images/sliceOverview.png similarity index 100% rename from g3doc/images/sliceOverview.png rename to docs/images/sliceOverview.png diff --git a/g3doc/images/sliceOverviewAfterFiltering.png b/docs/images/sliceOverviewAfterFiltering.png similarity index 100% rename from g3doc/images/sliceOverviewAfterFiltering.png rename to docs/images/sliceOverviewAfterFiltering.png diff --git a/docs/images/tf_full_color_primary_icon.svg b/docs/images/tf_full_color_primary_icon.svg new file mode 100644 index 0000000000..3e7247778d --- /dev/null +++ b/docs/images/tf_full_color_primary_icon.svg @@ -0,0 +1 @@ +FullColorPrimary Icon \ No newline at end of file diff --git a/g3doc/images/tfma-slicing-metrics-browser.gif b/docs/images/tfma-slicing-metrics-browser.gif similarity index 100% rename from g3doc/images/tfma-slicing-metrics-browser.gif rename to docs/images/tfma-slicing-metrics-browser.gif diff --git a/g3doc/images/tfma-slicing-metrics-browser.png b/docs/images/tfma-slicing-metrics-browser.png similarity index 100% rename from g3doc/images/tfma-slicing-metrics-browser.png rename to docs/images/tfma-slicing-metrics-browser.png diff --git a/g3doc/images/tfma_pipeline.png b/docs/images/tfma_pipeline.png similarity index 100% rename from g3doc/images/tfma_pipeline.png rename to docs/images/tfma_pipeline.png diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000000..035c6244a7 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,39 @@ +# Improving Model Quality With TensorFlow Model Analysis + +## Introduction + +As you tweak your model during development, you need to check whether your +changes are improving your model. Just checking accuracy may not be enough. For +example, if you have a classifier for a problem in which 95% of your instances +are positive, you may be able to improve accuracy by simply always predicting +positive, but you won't have a very robust classifier. + +## Overview + +The goal of TensorFlow Model Analysis is to provide a mechanism for model +evaluation in TFX. TensorFlow Model Analysis allows you to perform model +evaluations in the TFX pipeline, and view resultant metrics and plots in a +Jupyter notebook. Specifically, it can provide: + +* [Metrics](https://tensorflow.github.io/model-analysis/metrics) computed on entire training and holdout + dataset, as well as next-day evaluations +* Tracking metrics over time +* Model quality performance on different feature slices +* [Model validation](https://tensorflow.github.io/model-analysis/model_validations) for ensuring that + model's maintain consistent performance + +## Next Steps + +Try our [TFMA tutorial](https://tensorflow.github.io/tfx/tutorials/model_analysis/tfma_basic). + +Check out our [github](https://github.com/tensorflow/model-analysis) page for +details on the supported +[metrics and plots](https://tensorflow.github.io/model-analysis/metrics) and associated notebook +[visualizations](https://tensorflow.github.io/model-analysis/visualizations). + +See the [installation](https://tensorflow.github.io/model-analysis/install) and +[getting started](https://tensorflow.github.io/model-analysis/get_started) guides for information and +examples on how to get [set up](https://tensorflow.github.io/model-analysis/setup) in a standalone +pipeline. Recall that TFMA is also used within the [Evaluator](evaluator.md) +component in TFX, so these resources will be useful for getting started in TFX +as well. diff --git a/g3doc/install.md b/docs/install.md similarity index 100% rename from g3doc/install.md rename to docs/install.md diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 0000000000..0be88e0419 --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,19 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex" + } +}; + +document$.subscribe(() => { + MathJax.startup.output.clearCache() + MathJax.typesetClear() + MathJax.texReset() + MathJax.typesetPromise() +}) diff --git a/g3doc/metrics.md b/docs/metrics.md similarity index 97% rename from g3doc/metrics.md rename to docs/metrics.md index 3af92aaffa..c8b7103208 100644 --- a/g3doc/metrics.md +++ b/docs/metrics.md @@ -10,13 +10,13 @@ TFMA supports the following metrics and plots: are computed outside of the graph in beam using the metrics classes directly. * Standard TFMA metrics and plots - ([`tfma.metrics.*`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/metrics)) + ([`tfma.metrics.*`](../api_docs/python/tfma-metrics)) * Custom keras metrics (metrics derived from [`tf.keras.metrics.Metric`](https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Metric)) * Custom TFMA metrics (metrics derived from - [`tfma.metrics.Metric`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/metrics/Metric)) + [`tfma.metrics.Metric`](../api_docs/python/tfma-metrics#tensorflow_model_analysis.metrics.Metric)) using custom beam combiners or metrics derived from other metrics). NOTE: In TFMA, plots and metrics are both defined under the metrics library. By @@ -38,10 +38,10 @@ classification, ranking, etc. ## Configuration There are two ways to configure metrics in TFMA: (1) using the -[`tfma.MetricsSpec`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/api/MetricsSpec) +[`tfma.MetricsSpec`](../api_docs/python/tfma#tensorflow_model_analysis.metricsSpec) or (2) by creating instances of `tf.keras.metrics.*` and/or `tfma.metrics.*` classes in python and using -[`tfma.metrics.specs_from_metrics`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/api/metrics/specs_from_metrics) +[`tfma.metrics.specs_from_metrics`](../api_docs/python/tfma-metrics#tensorflow_model_analysis.metrics.specs_from_metrics) to convert them to a list of `tfma.MetricsSpec`. The following sections describe example configurations for different types of @@ -598,7 +598,7 @@ All the supported plots are stored in a single proto called ### EvalResult The return from an evaluation run is an -[`tfma.EvalResult`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/EvalResult). +[`tfma.EvalResult`](../api_docs/python/tfma#tensorflow_model_analysis.EvalResult). This record contains `slicing_metrics` that encode the metric key as a multi-level dict where the levels correspond to output name, class ID, metric name, and metric value respectively. This is intended to be used for UI display @@ -702,7 +702,7 @@ will be used by the combiner (see [architecture](architecture.md) for more info on what are extracts). All preprocessors will be executed sequentially in the order of the list. If the `preprocessors` is empty, then the combiner will be passed -[StandardMetricInputs](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/metrics/StandardMetricInputs) +[StandardMetricInputs](../api_docs/python/tfma-metrics#tensorflow_model_analysis.metrics.StandardMetricInputs) (standard metric inputs contains labels, predictions, and example_weights). The `combiner` is a `beam.CombineFn` that takes a tuple of (slice key, preprocessor output) as its input and outputs a tuple of (slice_key, metric results dict) as @@ -713,7 +713,7 @@ Note that slicing happens between the `preprocessors` and `combiner`. Note that if a metric computation wants to make use of both the standard metric inputs, but augment it with a few of the features from the `features` extracts, then the special -[FeaturePreprocessor](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/metrics/FeaturePreprocessor) +[FeaturePreprocessor](../api_docs/python/tfma-metrics#tensorflow_model_analysis.metrics.FeaturePreprocessor) can be used which will merge the requested features from multiple combiners into a single shared StandardMetricsInputs value that is passed to all the combiners (the combiners are responsible for reading the features they are interested in diff --git a/g3doc/model_validations.md b/docs/model_validations.md similarity index 100% rename from g3doc/model_validations.md rename to docs/model_validations.md diff --git a/g3doc/post_export_metrics.md b/docs/post_export_metrics.md similarity index 100% rename from g3doc/post_export_metrics.md rename to docs/post_export_metrics.md diff --git a/g3doc/setup.md b/docs/setup.md similarity index 100% rename from g3doc/setup.md rename to docs/setup.md diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css new file mode 100644 index 0000000000..21c97aa98c --- /dev/null +++ b/docs/stylesheets/extra.css @@ -0,0 +1,42 @@ +:root { + --md-primary-fg-color: #FFA800; + --md-primary-fg-color--light: #CCCCCC; + --md-primary-fg-color--dark: #425066; +} + +.video-wrapper { + max-width: 240px; + display: flex; + flex-direction: row; +} +.video-wrapper > iframe { + width: 100%; + aspect-ratio: 16 / 9; +} + +.buttons-wrapper { + flex-wrap: wrap; + gap: 1em; + display: flex; + /* flex-grow: 1; */ + /* justify-content: center; */ + /* align-content: center; */ +} + +.buttons-wrapper > a { + justify-content: center; + align-content: center; + flex-wrap: nowrap; + /* gap: 1em; */ + align-items: center; + text-align: center; + flex: 1 1 30%; + display: flex; +} + +.md-button > .buttons-content { + align-items: center; + justify-content: center; + display: flex; + gap: 1em; +} diff --git a/g3doc/visualizations.md b/docs/visualizations.md similarity index 79% rename from g3doc/visualizations.md rename to docs/visualizations.md index f93810b588..cca13730fe 100644 --- a/g3doc/visualizations.md +++ b/docs/visualizations.md @@ -1,17 +1,17 @@ # TensorFlow Model Analysis Visualizations The output of running an evaluation is a -[`tfma.EvalResult`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/EvalResult) +[`tfma.EvalResult`](../api_docs/python/tfma/#tensorflow_model_analysis.EvalResult) which can be visualized in a Jupyter notebook by calling -[`tfma.view.render_slicing_metrics`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/view/render_slicing_metrics) +[`tfma.view.render_slicing_metrics`](../api_docs/python/tfma-view#tensorflow_model_analysis.view.render_slicing_metrics) (or -[`tfma.view.render_plot`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/view/render_plot) +[`tfma.view.render_plot`](../api_docs/python/tfma-view#tensorflow_model_analysis.view.render_plot) for plots). ### Metrics View To view metrics, use the -[`tfma.view.render_slicing_metrics`] (https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/view/render_slicing_metrics) +[`tfma.view.render_slicing_metrics`](../api_docs/python/tfma-view/#tensorflow_model_analysis.view.render_slicing_metrics) API passing the `tfma.EvalResult` that was output from the evaluation run. The metrics view is composed of three parts: @@ -76,9 +76,9 @@ metrics view is composed of three parts: Each plot has it own visualization that is unique to the plot. For more information, see the relevant API documentation for the plot class. Note that -in TFMA, plots and metrics are both defined under [`tfma.metrics.*`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/api/metrics) +in TFMA, plots and metrics are both defined under [`tfma.metrics.*`](../api_docs/python/tfma-metrics) By convention the classes related to plots end in `Plot`. To view plots, use the -[`tfma.view.render_plot`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/view/render_plot) +[`tfma.view.render_plot`](../api_docs/python/tfma-view/#tensorflow_model_analysis.view.render_plot) API passing the `tfma.EvalResult` that was output from the evaluation run. ### Time Series Graphs @@ -87,9 +87,9 @@ Time series graphs make it easy to spot trends of a specific metric over data spans or model runs. To create a time series graph, perform multiple evaluations (saving the output to different directories), and then load them into a `tfma.EvalResults` object by calling -[`tfma.load_eval_results`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/api/load_eval_results). +[`tfma.load_eval_results`](../api_docs/python/tfma/#tensorflow_model_analysis.load_eval_results). The results can then be displayed using -[`tfma.view.render_time_series`](https://www.tensorflow.org/tfx/model_analysis/api_docs/python/tfma/view/render_time_series) +[`tfma.view.render_time_series`](../api_docs/python/tfma-view#tensorflow_model_analysis.view.render_time_series) To display the graph for a specific metric, simply click on it from the dropdown list. To dismiss a graph, click on the X on the upper right corner. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000000..246830d7ed --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,144 @@ +site_name: TensorFlow Model Analysis +repo_name: "model-analysis" +repo_url: https://github.com/tensorflow/model-analysis + +theme: + logo: images/tf_full_color_primary_icon.svg + name: material + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + primary: custom + accent: custom + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + primary: custom + accent: custom + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + primary: custom + accent: custom + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to system preference + favicon: images/tf_full_color_primary_icon.svg + + features: + - content.code.copy + - content.code.select + - content.action.edit + +plugins: + - search + - autorefs + - mkdocstrings: + default_handler: python + handlers: + python: + options: + show_source: true + show_root_heading: true + unwrap_annotated: true + show_symbol_type_toc: true + show_if_no_docstring: true + show_symbol_type_heading: true + merge_init_into_class: true + show_signature_annotations: true + separate_signature: true + signature_crossrefs: true + group_by_category: true + show_category_heading: true + show_submodules: false + show_root_full_path: true + docstring_section_style: "spacy" + inherited_members: true + summary: false + filters: + - "!^_" + - "^__init__$" + - "^__call__$" + - "!^logger" + extensions: + - griffe_inherited_docstrings + import: + - https://docs.python.org/3/objects.inv + +extra_css: + - stylesheets/extra.css + +extra_javascript: + - javascripts/mathjax.js + - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js + +markdown_extensions: + - admonition + - attr_list + - def_list + - tables + - toc: + permalink: true + - pymdownx.highlight: + anchor_linenums: true + linenums: false + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.arithmatex: + generic: true + - pymdownx.critic + - pymdownx.caret + - pymdownx.keys + - pymdownx.mark + - pymdownx.tilde + - md_in_html + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg +watch: + - tensorflow_model_analysis + +nav: + - Guide: + - Improving Model Quality: index.md + - Install: install.md + - Get Started: get_started.md + - Setup: setup.md + - Metrics and Plots: metrics.md + - Visualizations: visualizations.md + - Model Validations: model_validations.md + - Using Fairness Indicators: https://tensorflow.github.io/tfx/guide/fairness_indicators/ + - Using Fairness Indicators with Pandas DataFrames: https://tensorflow.github.io/fairness-indicators/tutorials/Fairness_Indicators_Pandas_Case_Study/ + - Architecture: architecture.md + - FAQ: faq.md + - API: + - tfma: api_docs/python/tfma.md + # - tfma.addons: api_docs/python/tfma-addons.md + - tfma.constants: api_docs/python/tfma-constants.md + - tfma.contrib: api_docs/python/tfma-contrib.md + - tfma.evaluators: api_docs/python/tfma-evaluators.md + - tfma.experimental 🧪: api_docs/python/tfma-experimental.md + # - tfma.export: api_docs/python/tfma-export.md + # - tfma.exporter: api_docs/python/tfma-exporter.md + - tfma.extractors: api_docs/python/tfma-extractors.md + - tfma.metrics: api_docs/python/tfma-metrics.md + # - tfma.model_agnostic_eval: api_docs/python/tfma-model_agnostic_eval.md + - tfma.post_export_metrics: api_docs/python/tfma-post_export_metrics.md + - tfma.sdk: api_docs/python/tfma-sdk.md + - tfma.types: api_docs/python/tfma-types.md + - tfma.utils: api_docs/python/tfma-utils.md + - tfma.validators: api_docs/python/tfma-validators.md + - tfma.version: api_docs/python/tfma-version.md + - tfma.view: api_docs/python/tfma-view.md + - tfma.writers: api_docs/python/tfma-writers.md diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000000..15b3414693 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,6 @@ +mkdocs +mkdocs-material +mkdocstrings[python] +griffe-inherited-docstrings +mkdocs-autorefs +ruff diff --git a/setup.py b/setup.py index 28814fd0e8..0ffeab3f74 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ from distutils import log from distutils import spawn import os +from pathlib import Path import platform import subprocess import sys @@ -249,6 +250,15 @@ def run(self): # update package data in case this created new files update_package_data(self.distribution) +def _make_docs_packages(): + return [ + req for req in Path("./requirements-docs.txt") + .expanduser() + .resolve() + .read_text() + .splitlines() + if req + ] def _make_extra_packages_tfjs(): # Packages needed for tfjs. @@ -327,7 +337,8 @@ def select_constraint(default, nightly=None, git_master=None): ), ], 'extras_require': { - 'all': _make_extra_packages_tfjs(), + 'all': [*_make_extra_packages_tfjs(), *_make_docs_packages()], + 'docs': _make_docs_packages(), }, 'python_requires': '>=3.9,<4', 'packages': find_packages(), diff --git a/tensorflow_model_analysis/__init__.py b/tensorflow_model_analysis/__init__.py index 83e2c5f781..f130fc578b 100644 --- a/tensorflow_model_analysis/__init__.py +++ b/tensorflow_model_analysis/__init__.py @@ -112,3 +112,68 @@ def _jupyter_nbextension_paths(): 'dest': 'tensorflow_model_analysis', 'require': 'tensorflow_model_analysis/extension' }] + +__all__ = [ + 'AddMetricsCallbackType', + 'AggregationOptions', + 'analyze_raw_data', + 'AttributionsForSlice', + 'BatchedInputsToExtracts', + 'BinarizationOptions', + 'ConfidenceIntervalOptions', + 'CrossSliceMetricThreshold', + 'CrossSliceMetricThresholds', + 'CrossSlicingSpec', + 'default_eval_shared_model', + 'default_evaluators', + 'default_extractors', + 'default_writers', + 'EvalConfig', + 'EvalResult', + 'EvalSharedModel', + 'ExampleWeightOptions', + 'ExtractAndEvaluate', + 'ExtractEvaluateAndWriteResults', + 'Extracts', + 'FeaturesPredictionsLabels', + 'GenericChangeThreshold', + 'GenericValueThreshold', + 'InputsToExtracts', + 'is_batched_input', + 'is_legacy_estimator', + 'load_attributions', + 'load_eval_result', + 'load_eval_results', + 'load_metrics', + 'load_plots', + 'load_validation_result', + 'make_eval_results', + 'MaterializedColumn', + 'MaybeMultipleEvalSharedModels', + 'MetricConfig', + 'MetricsForSlice', + 'MetricsSpec', + 'MetricThreshold', + 'ModelLoader', + 'ModelSpec', + 'multiple_data_analysis', + 'multiple_model_analysis', + 'Options', + 'PaddingOptions', + 'PerSliceMetricThreshold', + 'PerSliceMetricThresholds', + 'PlotsForSlice', + 'RaggedTensorValue', + 'RepeatedInt32Value', + 'RepeatedStringValue', + 'run_model_analysis', + 'SlicingSpec', + 'SparseTensorValue', + 'TensorType', + 'TensorTypeMaybeDict', + 'TensorValue', + 'Validate', + 'ValidationResult', + 'VarLenTensorValue', + 'WriteResults' +] diff --git a/tensorflow_model_analysis/api/types.py b/tensorflow_model_analysis/api/types.py index b05e7ab603..81fe98fedd 100644 --- a/tensorflow_model_analysis/api/types.py +++ b/tensorflow_model_analysis/api/types.py @@ -640,3 +640,39 @@ def __new__( MaybeMultipleEvalSharedModels = Union[ EvalSharedModel, List[EvalSharedModel], Dict[str, EvalSharedModel] ] + +__all__ = [ + 'AddMetricsCallbackType', + 'ConcreteStructuredMetricValue', + 'DictOfFetchedTensorValues', + 'DictOfTensorType', + 'DictOfTensorTypeMaybeDict', + 'DictOfTensorValue', + 'DictOfTensorValueMaybeDict', + 'DictOfTypeSpec', + 'DictOfTypeSpecMaybeDict', + 'EvalSharedModel', + 'Extracts', + 'FeaturesPredictionsLabels', + 'FPLKeyType', + 'MaterializedColumn', + 'MaybeMultipleEvalSharedModels', + 'MetricValueType', + 'MetricVariablesType', + 'ModelLoader', + 'PrimitiveMetricValueType', + 'RaggedTensorValue', + 'SparseTensorValue', + 'StructuredMetricValue', + 'TensorOrOperationType', + 'TensorType', + 'TensorTypeMaybeDict', + 'TensorTypeMaybeMultiLevelDict', + 'TensorValue', + 'TensorValueMaybeDict', + 'TensorValueMaybeMultiLevelDict', + 'TypeSpecMaybeDict', + 'TypeSpecMaybeMultiLevelDict', + 'ValueWithTDistribution', + 'VarLenTensorValue' +] diff --git a/tensorflow_model_analysis/evaluators/__init__.py b/tensorflow_model_analysis/evaluators/__init__.py index 4cc1a0255c..fa774c312e 100644 --- a/tensorflow_model_analysis/evaluators/__init__.py +++ b/tensorflow_model_analysis/evaluators/__init__.py @@ -19,3 +19,12 @@ from tensorflow_model_analysis.evaluators.evaluator import Evaluator from tensorflow_model_analysis.evaluators.evaluator import verify_evaluator from tensorflow_model_analysis.evaluators.metrics_plots_and_validations_evaluator import MetricsPlotsAndValidationsEvaluator + +__all__ = [ + "AnalysisTableEvaluator", + "Evaluation", + "Evaluator" + "MetricsAndPlotsEvaluator", + "MetricsPlotsAndValidationsEvaluator", + "verify_evaluator", +] diff --git a/tensorflow_model_analysis/extractors/__init__.py b/tensorflow_model_analysis/extractors/__init__.py index 2349fba69c..171684df43 100644 --- a/tensorflow_model_analysis/extractors/__init__.py +++ b/tensorflow_model_analysis/extractors/__init__.py @@ -27,3 +27,19 @@ from tensorflow_model_analysis.extractors.slice_key_extractor import SliceKeyExtractor from tensorflow_model_analysis.extractors.transformed_features_extractor import TransformedFeaturesExtractor from tensorflow_model_analysis.extractors.unbatch_extractor import UnbatchExtractor + +__all__ = [ + "ExampleWeightsExtractor", + "Extractor", + "FeatureExtractor", + "FeaturesExtractor", + "Filter", + "InputExtractor", + "LabelsExtractor", + "PredictExtractor", + "PredictionsExtractor", + "SliceKeyExtractor", + "SLICE_KEY_EXTRACTOR_STAGE_NAME", + "TransformedFeaturesExtractor", + "UnbatchExtractor", +] diff --git a/tensorflow_model_analysis/metrics/__init__.py b/tensorflow_model_analysis/metrics/__init__.py index a00066158f..eb2a04fe56 100644 --- a/tensorflow_model_analysis/metrics/__init__.py +++ b/tensorflow_model_analysis/metrics/__init__.py @@ -138,3 +138,120 @@ from tensorflow_model_analysis.metrics.weighted_example_count import WeightedExampleCount # TODO(b/143180976): Remove WeightedExampleCount. + +__all__ = [ + 'AttributionsMetric', + 'AUC', + 'AUCCurve', + 'AUCPrecisionRecall', + 'AUCSummationMethod', + 'BalancedAccuracy', + 'BinaryAccuracy', + 'BinaryCrossEntropy', + 'BooleanFlipRates', + 'Calibration', + 'CalibrationPlot', + 'CategoricalCrossEntropy', + 'COCOAveragePrecision', + 'COCOAverageRecall', + 'COCOMeanAveragePrecision', + 'COCOMeanAverageRecall', + 'CoefficientOfDiscrimination', + 'CombinedFeaturePreprocessor', + 'ConfusionMatrixAtThresholds', + 'ConfusionMatrixPlot', + 'default_binary_classification_specs', + 'default_multi_class_classification_specs', + 'default_regression_specs', + 'DerivedMetricComputation', + 'DiagnosticOddsRatio', + 'ExactMatch', + 'ExampleCount', + 'F1Score', + 'FallOut', + 'FalseDiscoveryRate', + 'FalseNegatives', + 'FalseOmissionRate', + 'FalsePositives', + 'FeaturePreprocessor', + 'FN', + 'FNR', + 'FowlkesMallowsIndex', + 'FP', + 'FPR', + 'has_attributions_metrics', + 'Informedness', + 'Markedness', + 'MatthewsCorrelationCoefficient', + 'MaxRecall', + 'Mean', + 'MeanAbsoluteAttributions', + 'MeanAbsoluteError', + 'MeanAbsolutePercentageError', + 'MeanAttributions', + 'MeanLabel', + 'MeanPrediction', + 'MeanSquaredError', + 'MeanSquaredLogarithmicError', + 'merge_per_key_computations', + 'Metric', + 'metric_thresholds_from_metrics_specs', + 'MetricComputation', + 'MetricComputations', + 'MetricKey', + 'MetricsDict', + 'MinLabelPosition', + 'MissRate', + 'MultiClassConfusionMatrixAtThresholds', + 'MultiClassConfusionMatrixPlot', + 'MultiLabelConfusionMatrixPlot', + 'NDCG', + 'NegativeLikelihoodRatio', + 'NegativePredictiveValue', + 'NO_PREDICTED_CLASS_ID', + 'NPV', + 'ObjectDetectionConfusionMatrixPlot', + 'ObjectDetectionMaxRecall', + 'ObjectDetectionPrecision', + 'ObjectDetectionPrecisionAtRecall', + 'ObjectDetectionRecall', + 'ObjectDetectionThresholdAtRecall', + 'PlotKey', + 'PositiveLikelihoodRatio', + 'PPV', + 'Precision', + 'PrecisionAtRecall', + 'Preprocessor', + 'Prevalence', + 'PrevalenceThreshold', + 'QueryStatistics', + 'Recall', + 'RecallAtPrecision', + 'RelativeCoefficientOfDiscrimination', + 'ScoreDistributionPlot', + 'SemanticSegmentationConfusionMatrix', + 'SemanticSegmentationFalsePositive', + 'SemanticSegmentationTruePositive', + 'SensitivityAtSpecificity', + 'SetMatchPrecision', + 'SetMatchRecall', + 'Specificity', + 'SpecificityAtSensitivity', + 'specs_from_metrics', + 'SquaredPearsonCorrelation', + 'StandardMetricInputs', + 'SubKey', + 'SymmetricPredictionDifference', + 'ThreatScore', + 'TN', + 'TNR', + 'to_label_prediction_example_weight', + 'to_standard_metric_inputs', + 'TotalAbsoluteAttributions', + 'TotalAttributions', + 'TP', + 'TPR', + 'TrueNegatives', + 'TruePositives', + 'WeightedExampleCount' +] diff --git a/tensorflow_model_analysis/post_export_metrics/__init__.py b/tensorflow_model_analysis/post_export_metrics/__init__.py index 8a637fbce9..887163ad00 100644 --- a/tensorflow_model_analysis/post_export_metrics/__init__.py +++ b/tensorflow_model_analysis/post_export_metrics/__init__.py @@ -18,3 +18,23 @@ """ from tensorflow_model_analysis.post_export_metrics import metric_keys +from tensorflow_model_analysis.post_export_metrics import * + +__all__ = [ + "auc", + "auc_plots", + "calibration", + "calibration_plot_and_prediction_histogram", + "confusion_matrix_at_thresholds", + "DEFAULT_KEY_PREFERENCE", + "example_count", + "example_weight", + "fairness_auc", + "fairness_indicators", + "mean_absolute_error", + "mean_squared_error", + "precision_at_k", + "recall_at_k", + "root_mean_squared_error", + "squared_pearson_correlation", +] diff --git a/tensorflow_model_analysis/sdk.py b/tensorflow_model_analysis/sdk.py index 6bdc334cdd..14f15fd3c9 100644 --- a/tensorflow_model_analysis/sdk.py +++ b/tensorflow_model_analysis/sdk.py @@ -83,3 +83,34 @@ # Import VERSION as VERSION_STRING for backwards compatibility. from tensorflow_model_analysis.version import VERSION as VERSION_STRING + +__all__ = [ + "ANALYSIS_KEY", + "ARROW_INPUT_COLUMN", + "ARROW_RECORD_BATCH_KEY", + "ATTRIBUTIONS_KEY", + "BASELINE_KEY", + "BASELINE_SCORE_KEY", + "CANDIDATE_KEY", + "DATA_CENTRIC_MODE", + "EXAMPLE_SCORE_KEY", + "EXAMPLE_WEIGHTS_KEY", + "FEATURES_KEY", + "FEATURES_PREDICTIONS_LABELS_KEY", + "INPUT_KEY", + "LABELS_KEY", + "METRICS_KEY", + "MODEL_CENTRIC_MODE", + "MetricDirection", + "PLOTS_KEY", + "PREDICTIONS_KEY", + "SLICE_KEY_TYPES_KEY", + "TFMA_EVAL", + "TF_ESTIMATOR", + "TF_GENERIC", + "TF_JS", + "TF_KERAS", + "TF_LITE", + "VALIDATIONS_KEY", + "VERSION_STRING", +] diff --git a/tensorflow_model_analysis/utils/__init__.py b/tensorflow_model_analysis/utils/__init__.py index 9cf6e71237..eb42c00659 100644 --- a/tensorflow_model_analysis/utils/__init__.py +++ b/tensorflow_model_analysis/utils/__init__.py @@ -32,3 +32,24 @@ from tensorflow_model_analysis.utils.util import get_by_keys from tensorflow_model_analysis.utils.util import merge_extracts from tensorflow_model_analysis.utils.util import unique_key + +__all__ = [ + "calculate_confidence_interval", + "CombineFnWithModels", + "compound_key", + "create_keys_key", + "create_values_key", + "DoFnWithModels", + "get_baseline_model_spec", + "get_by_keys", + "get_model_spec", + "get_model_type", + "get_non_baseline_model_specs", + "has_change_threshold", + "merge_extracts", + "model_construct_fn", + "unique_key", + "update_eval_config_with_defaults", + "verify_and_update_eval_shared_models", + "verify_eval_config", +] diff --git a/tensorflow_model_analysis/validators/__init__.py b/tensorflow_model_analysis/validators/__init__.py index 97acea7df2..2fbd2f7423 100644 --- a/tensorflow_model_analysis/validators/__init__.py +++ b/tensorflow_model_analysis/validators/__init__.py @@ -15,3 +15,8 @@ from tensorflow_model_analysis.validators.validator import Validation from tensorflow_model_analysis.validators.validator import Validator + +__all__ = [ + "Validation", + "Validator", +] diff --git a/tensorflow_model_analysis/view/__init__.py b/tensorflow_model_analysis/view/__init__.py index c96911a175..e00ae6925e 100644 --- a/tensorflow_model_analysis/view/__init__.py +++ b/tensorflow_model_analysis/view/__init__.py @@ -19,3 +19,12 @@ from tensorflow_model_analysis.view.widget_view import render_slicing_attributions from tensorflow_model_analysis.view.widget_view import render_slicing_metrics from tensorflow_model_analysis.view.widget_view import render_time_series + +__all__ = [ + "SlicedMetrics", + "SlicedPlots", + "render_plot", + "render_slicing_attributions", + "render_slicing_metrics", + "render_time_series", +] diff --git a/tensorflow_model_analysis/writers/__init__.py b/tensorflow_model_analysis/writers/__init__.py index 2494f43bfa..18d55ed7ba 100644 --- a/tensorflow_model_analysis/writers/__init__.py +++ b/tensorflow_model_analysis/writers/__init__.py @@ -18,3 +18,11 @@ from tensorflow_model_analysis.writers.metrics_plots_and_validations_writer import MetricsPlotsAndValidationsWriter from tensorflow_model_analysis.writers.writer import Write from tensorflow_model_analysis.writers.writer import Writer + +__all__ = [ + "EvalConfigWriter", + "MetricsPlotsAndValidationsWriter", + "Write", + "Writer", + "convert_slice_metrics_to_proto", +]